diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/AsyncLatch.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/AsyncLatch.java new file mode 100644 index 00000000..b28b804d --- /dev/null +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/AsyncLatch.java @@ -0,0 +1,65 @@ +package com.launchdarkly.sdk.server; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.WeakHashMap; +import java.util.concurrent.CompletableFuture; + +/** + * A latch that is released once, and that callers wait on through futures. + *

+ * This exists so that a loop can wait for "a result or the latch, whichever comes first" on every iteration without + * a memory leak. The obvious way to write that wait is {@code CompletableFuture.anyOf(result, latchFuture)}. But + * {@code anyOf} attaches a completion node to each future it is given, and a pending {@link CompletableFuture} keeps + * those nodes until it completes. If the latch used one shared future, every iteration that ended with a result would + * leave one more node on that future. The latch can stay pending for the life of the process, so the nodes would + * accumulate without limit. + *

+ * To prevent this, {@link #newFuture()} returns a new future for each wait. The latch holds these futures weakly, so a + * future that nobody waits on any more is garbage collected, together with the nodes that {@code anyOf} attached to + * it. When the latch is released, it completes each future that is still waiting. After the release, + * {@link #newFuture()} returns a future that is already complete. + */ +final class AsyncLatch { + private final Object lock = new Object(); + + // The futures that wait for the release, or null after the release. + private Set> pending = Collections.newSetFromMap(new WeakHashMap<>()); + + /** + * A new future that completes when the latch is released. + * + * @return the future, which is already complete if the latch has been released + */ + CompletableFuture newFuture() { + CompletableFuture future = new CompletableFuture<>(); + synchronized (lock) { + if (pending != null) { + pending.add(future); + return future; + } + } + future.complete(null); + return future; + } + + /** + * Releases the latch, which completes every future from {@link #newFuture()}. Releasing it again has no effect. + */ + void release() { + List> waiting; + synchronized (lock) { + if (pending == null) { + return; + } + // The list holds the futures strongly, so none is garbage collected before it completes. + waiting = new ArrayList<>(pending); + pending = null; + } + for (CompletableFuture future : waiting) { + future.complete(null); + } + } +} diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java index ab98fde5..e80c4a41 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapter.java @@ -1,6 +1,8 @@ package com.launchdarkly.sdk.server; import com.launchdarkly.sdk.internal.collections.IterableAsyncQueue; +import com.launchdarkly.sdk.internal.http.FailureClass; +import com.launchdarkly.sdk.internal.http.HttpErrors; import com.launchdarkly.sdk.fdv2.Selector; import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; import com.launchdarkly.sdk.server.datasources.Synchronizer; @@ -130,9 +132,14 @@ public void close() { * This sink doesn't delegate to any other sink - it exists solely to convert FDv1 updates to FDv2 results. */ private static class ConvertingUpdateSink implements DataSourceUpdateSink { + // The number of VALID statuses in a row that meet the reset threshold, the same as for a polling synchronizer. + private static final int RESET_THRESHOLD_SUCCESSFUL_RESULTS = 2; + private final IterableAsyncQueue resultQueue; // Only a full data set carries an environment ID; it is retained for subsequent partial updates. private volatile String environmentId = null; + // The number of VALID statuses the data source has reported since its last failure. + private int consecutiveSuccessfulResults = 0; public ConvertingUpdateSink(IterableAsyncQueue resultQueue) { this.resultQueue = resultQueue; @@ -189,16 +196,39 @@ public void updateStatus(DataSourceStatusProvider.State newState, DataSourceStat // Convert state changes to FDv2SourceResult status events switch (newState) { case INTERRUPTED: - resultQueue.put(FDv2SourceResult.interrupted(newError, false)); + consecutiveSuccessfulResults = 0; + if (newError != null + && newError.getKind() == DataSourceStatusProvider.ErrorKind.ERROR_RESPONSE + && HttpErrors.classifyHttpFailure(newError.getStatusCode()) == FailureClass.UNEXPECTED) { + resultQueue.put(FDv2SourceResult.terminalError(newError, false)); + } else { + resultQueue.put(FDv2SourceResult.interrupted(newError, false)); + } break; case OFF: + consecutiveSuccessfulResults = 0; if (newError != null) { resultQueue.put(FDv2SourceResult.terminalError(newError, false)); + } else { + resultQueue.put(FDv2SourceResult.shutdown()); } break; case VALID: + consecutiveSuccessfulResults++; + if (consecutiveSuccessfulResults == RESET_THRESHOLD_SUCCESSFUL_RESULTS) { + // Any data that came with this status is already queued, so report the reset with no changes. + ChangeSet>>> noChanges = + new ChangeSet<>( + ChangeSetType.None, + Selector.EMPTY, + Collections.emptyList(), + environmentId, + /* shouldPersist= */ true); + resultQueue.put(FDv2SourceResult.changeSet(noChanges, false).withResetThresholdMet()); + } + break; case INITIALIZING: - // These states don't map to FDv2SourceResult status events + // This state doesn't map to an FDv2SourceResult status event break; } } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSystemComponents.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSystemComponents.java index 51526036..0b770ecc 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSystemComponents.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/DataSystemComponents.java @@ -105,8 +105,10 @@ public Synchronizer build(DataSourceBuildInputs context) { StandardEndpoints.FDV2_STREAMING_REQUEST_PATH, context.getBaseLogger(), context.getSelectorSource(), + context.getSharedExecutor(), payloadFilter, initialReconnectDelay, + StreamingSynchronizerImpl.RESET_THRESHOLD, context.getThreadPriority(), context.getDiagnosticStore() ); diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java index 360be672..de6dffa0 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSource.java @@ -11,35 +11,59 @@ import com.launchdarkly.sdk.server.subsystems.DataSource; import com.launchdarkly.sdk.server.subsystems.DataSourceUpdateSinkV2; +import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; -import java.util.Collections; import java.util.Date; import java.util.List; -import java.util.Set; -import java.util.WeakHashMap; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; -import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.Condition; -import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.ConditionFactory; -import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.FallbackCondition; -import static com.launchdarkly.sdk.server.FDv2DataSourceConditions.RecoveryCondition; - class FDv2DataSource implements DataSource { /** - * Default fallback timeout of 2 minutes. The timeout is only configurable for testing. + * Default fallback timeout of 2 minutes. + */ + private static final Duration DEFAULT_FALLBACK_TIMEOUT = Duration.ofMinutes(2); + + /** + * Default recovery timeout of 5 minutes. */ - private static final int defaultFallbackTimeoutSeconds = 2 * 60; + private static final Duration DEFAULT_RECOVERY_TIMEOUT = Duration.ofMinutes(5); /** - * Default recovery timeout of 5 minutes. The timeout is only configurable for testing. + * Default base of the park delay that follows an unexpected failure. */ - private static final long defaultRecoveryTimeout = 5 * 60; + private static final Duration DEFAULT_PARK_INITIAL_DELAY = Duration.ofMinutes(5); + + /** + * Default maximum of the park delay. + */ + private static final Duration DEFAULT_PARK_MAX_DELAY = Duration.ofHours(1); + + /** + * Default base of the failover delay that follows a park. + */ + private static final Duration DEFAULT_FAILOVER_DELAY = Duration.ofSeconds(1); + + /** + * Default duration of a fallback to FDv1, when the directive does not give one. + */ + private static final Duration DEFAULT_FDV1_FALLBACK_TTL = Duration.ofHours(1); + + /** + * The longest time that close waits for the data source to stop. + */ + private static final Duration CLOSE_TIMEOUT = Duration.ofSeconds(5); private final SourceManager sourceManager; - private final List conditionFactories; + private final Duration fallbackTimeout; + private final Duration recoveryTimeout; + private final Duration parkInitialDelay; + private final Duration parkMaxDelay; + private final Duration failoverDelay; + private final Duration defaultFDv1FallbackTTL; private final DataSourceUpdateSinkV2 dataSourceUpdates; @@ -52,6 +76,16 @@ class FDv2DataSource implements DataSource { private volatile boolean closed = false; + /** + * Released when the data source is closed. + */ + private final AsyncLatch closeLatch = new AsyncLatch(); + + /** + * The thread that runs the initializers and synchronizers, or null if the data source has not started. + */ + private volatile Thread runThread; + /** * Avoid duplicate orchestration logs for the same synchronizer and {@link SourceSignal}. */ @@ -64,48 +98,53 @@ public interface DataSourceFactory { } public FDv2DataSource( - ImmutableList> initializers, - ImmutableList> synchronizers, - DataSourceFactory fdv1DataSourceFactory, - DataSourceUpdateSinkV2 dataSourceUpdates, - int threadPriority, - LDLogger logger, - ScheduledExecutorService sharedExecutor - ) { + ImmutableList> initializers, + ImmutableList> synchronizers, + DataSourceFactory fdv1DataSourceFactory, + DataSourceUpdateSinkV2 dataSourceUpdates, + int threadPriority, + LDLogger logger) { this(initializers, - synchronizers, - fdv1DataSourceFactory, - dataSourceUpdates, - threadPriority, - logger, - sharedExecutor, - defaultFallbackTimeoutSeconds, - defaultRecoveryTimeout - ); + synchronizers, + fdv1DataSourceFactory, + dataSourceUpdates, + threadPriority, + logger, + DEFAULT_FALLBACK_TIMEOUT, + DEFAULT_RECOVERY_TIMEOUT, + DEFAULT_PARK_INITIAL_DELAY, + DEFAULT_PARK_MAX_DELAY, + DEFAULT_FAILOVER_DELAY, + DEFAULT_FDV1_FALLBACK_TTL); } + /** + * Creates a data source with the given timeouts and delays. Tests use this constructor to shorten them. + */ public FDv2DataSource( - ImmutableList> initializers, - ImmutableList> synchronizers, - DataSourceFactory fdv1DataSourceFactory, - DataSourceUpdateSinkV2 dataSourceUpdates, - int threadPriority, - LDLogger logger, - ScheduledExecutorService sharedExecutor, - long fallbackTimeout, - long recoveryTimeout - ) { + ImmutableList> initializers, + ImmutableList> synchronizers, + DataSourceFactory fdv1DataSourceFactory, + DataSourceUpdateSinkV2 dataSourceUpdates, + int threadPriority, + LDLogger logger, + Duration fallbackTimeout, + Duration recoveryTimeout, + Duration parkInitialDelay, + Duration parkMaxDelay, + Duration failoverDelay, + Duration defaultFDv1FallbackTTL) { List synchronizerFactories = synchronizers - .stream() - .map(SynchronizerFactoryWithState::new) - // Collect to an ArrayList to ensure mutability. - .collect(Collectors.toCollection(ArrayList::new)); + .stream() + .map(SynchronizerFactoryWithState::new) + // Collect to an ArrayList to ensure mutability. + .collect(Collectors.toCollection(ArrayList::new)); // If we have a fdv1 data source factory, then add that to the synchronizer factories in a blocked state. // If we receive a request to fallback, then we will unblock it and block all other synchronizers. if (fdv1DataSourceFactory != null) { SynchronizerFactoryWithState wrapped = new SynchronizerFactoryWithState(fdv1DataSourceFactory, - true); + true); wrapped.block(); synchronizerFactories.add(wrapped); @@ -117,19 +156,21 @@ public FDv2DataSource( this.dataSourceUpdates = dataSourceUpdates; this.threadPriority = threadPriority; this.logger = logger; - this.conditionFactories = new ArrayList<>(); - this.conditionFactories.add(new FallbackCondition.Factory(sharedExecutor, fallbackTimeout)); - this.conditionFactories.add(new RecoveryCondition.Factory(sharedExecutor, recoveryTimeout)); + this.fallbackTimeout = fallbackTimeout; + this.recoveryTimeout = recoveryTimeout; + this.parkInitialDelay = parkInitialDelay; + this.parkMaxDelay = parkMaxDelay; + this.failoverDelay = failoverDelay; + this.defaultFDv1FallbackTTL = defaultFDv1FallbackTTL; } private void run() { - Thread runThread = new Thread(() -> { + runThread = new Thread(() -> { if (!sourceManager.hasAvailableSources()) { // There are not any initializer or synchronizers, so we are at the best state that // can be achieved. logger.warn( - "LaunchDarkly client will not connect to LaunchDarkly for feature flag data due to no initializers or synchronizers configured." - ); + "LaunchDarkly client will not connect to LaunchDarkly for feature flag data due to no initializers or synchronizers configured."); dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.VALID, null); startFuture.complete(true); return; @@ -140,44 +181,25 @@ private void run() { initializerOutcome = runInitializers(); } - // If an initializer signalled FDv1 fallback, block every FDv2 synchronizer in - // one shot via fdv1Fallback() (which also unblocks the FDv1 fallback - // synchronizer, if one is configured). If FDv1 is configured we hand off to it; - // otherwise we halt the data system. This takes precedence over the standard - // synchronizer chain -- the FDv2 synchronizers are not given a chance to run. + // If an initializer signalled FDv1 fallback, fall back to FDv1 before any synchronizer runs. if (initializerOutcome.fallbackToFDv1) { - sourceManager.fdv1Fallback(); - if (sourceManager.hasFDv1Fallback()) { - logger.warn("Initializer requested fallback to FDv1; switching to FDv1 fallback synchronizer."); - } else { - logger.warn("Initializer requested fallback to FDv1, but no FDv1 fallback synchronizer is configured."); - dataSourceUpdates.updateStatus( - DataSourceStatusProvider.State.OFF, - initializerOutcome.errorInfo); - startFuture.complete(false); - return; - } + fallBackToFDv1(jittered(defaultFDv1FallbackTTL), initializerOutcome.errorInfo); } - if(!sourceManager.hasAvailableSynchronizers()) { + if (!sourceManager.hasFDv2Synchronizers() && !sourceManager.hasAvailableSynchronizers()) { // If already completed by the initializers, then this will have no effect. if (!isInitialized()) { // If we have no synchronizers, and we didn't manage to initialize, and we aren't shutting down, // then that was unexpected, and we will report it. - maybeReportUnexpectedExhaustion("All initializers exhausted and there are no available synchronizers."); + maybeReportUnexpectedExhaustion( + "All initializers exhausted and there are no available synchronizers."); } // If already completed has no effect. startFuture.complete(false); return; } - boolean haltedByDirective = runSynchronizers(); - - if (!haltedByDirective) { - // If we had synchronizers, and we ran out of them, and we aren't shutting down, then that was unexpected, - // and we will report it. - maybeReportUnexpectedExhaustion("All data source acquisition methods have been exhausted."); - } + runSynchronizers(); // If we had initialized at some point, then the future will already be complete and this will be ignored. startFuture.complete(false); @@ -215,11 +237,11 @@ private InitializerOutcome runInitializers() { logger.info("Initialized via '{}'.", initializerName); if (!result.getChangeSet().getSelector().isEmpty()) { // A defined selector marks initialization complete -- match Go/Python/Ruby - // behavior. A selectorless basis is applied so evaluations can serve it, - // and once the initializer chain is fully exhausted that applied data is - // also enough to consider initialization complete (see the post-loop + // behavior. A full payload without a selector is applied so evaluations can + // serve it, and once the initializer chain is fully exhausted that applied + // data is also enough to consider initialization complete (see the post-loop // block below); but mid-chain we don't yet flip to VALID, so a later - // initializer can still produce a selectorful basis if one is available. + // initializer can still produce a basis if one is available. dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.VALID, null); startFuture.complete(true); if (result.isFdv1Fallback()) { @@ -234,16 +256,16 @@ private InitializerOutcome runInitializers() { case INTERRUPTED: case TERMINAL_ERROR: logger.warn("Initializer '{}' failed: {}", - initializerName, - detailForError(status.getErrorInfo())); + initializerName, + detailForError(status.getErrorInfo())); fallbackErrorInfo = status.getErrorInfo(); // The data source updates handler will filter the state during initializing, but this // will make the error information available. dataSourceUpdates.updateStatus( - // While the error was terminal to the individual initializer, it isn't terminal - // to the data source as a whole. - DataSourceStatusProvider.State.INTERRUPTED, - status.getErrorInfo()); + // While the error was terminal to the individual initializer, it isn't terminal + // to the data source as a whole. + DataSourceStatusProvider.State.INTERRUPTED, + status.getErrorInfo()); break; case SHUTDOWN: case GOODBYE: @@ -257,7 +279,7 @@ private InitializerOutcome runInitializers() { // selector, so initialization is incomplete) or on a STATUS error result. // In either case, the SDK must halt the FDv2 chain immediately and switch // to the FDv1 fallback synchronizer; the eventual VALID status will come - // from the FDv1 synchronizer once it serves a selectorful payload. + // from the FDv1 synchronizer once it serves a full payload. if (result.isFdv1Fallback()) { return InitializerOutcome.fallbackToFDv1(fallbackErrorInfo); } @@ -266,24 +288,24 @@ private InitializerOutcome runInitializers() { // The data source updates handler will filter the state during initializing, but this // will make the error information available. dataSourceUpdates.updateStatus( - DataSourceStatusProvider.State.INTERRUPTED, - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.UNKNOWN, - 0, - e.toString(), - new Date().toInstant())); + DataSourceStatusProvider.State.INTERRUPTED, + new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.UNKNOWN, + 0, + e.toString(), + new Date().toInstant())); Throwable root = e instanceof ExecutionException && e.getCause() != null ? e.getCause() : e; logger.error("Error running initializer '{}': {}", - initializerName, - root.getMessage() != null ? root.getMessage() : LogValues.exceptionSummary(root)); + initializerName, + root.getMessage() != null ? root.getMessage() : LogValues.exceptionSummary(root)); } initializer = sourceManager.getNextInitializerAndSetActive(); } - // No initializer produced a selectorful basis, but at least one initializer applied - // a selectorless basis. Treat that as enough to consider the data system initialized + // No initializer produced a basis, but at least one initializer applied a full payload + // without a selector. Treat that as enough to consider the data system initialized // now that the entire initializer chain is exhausted -- evaluations can serve the // applied data, and the synchronizer phase (when configured) will continue from - // there. Without this, an SDK configured with only selectorless initializers and no - // synchronizer would never transition out of INITIALIZING. + // there. Without this, an SDK configured with only initializers that give no selector + // and no synchronizer would never transition out of INITIALIZING. if (anyDataReceived) { dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.VALID, null); startFuture.complete(true); @@ -313,98 +335,63 @@ static InitializerOutcome fallbackToFDv1(DataSourceStatusProvider.ErrorInfo erro } } - /** - * Determine conditions for the current synchronizer. Synchronizers require different conditions depending on if - * they are the 'prime' synchronizer or if there are other available synchronizers to use. - * - * @return a list of conditions to apply to the synchronizer - */ - private List getConditions() { - int availableSynchronizers = sourceManager.getAvailableSynchronizerCount(); - boolean isPrimeSynchronizer = sourceManager.isPrimeSynchronizer(); - - if (availableSynchronizers == 1) { - // If there is only 1 synchronizer, then we cannot fall back or recover, so we don't need any conditions. - return Collections.emptyList(); - } - if (isPrimeSynchronizer) { - // If there isn't a synchronizer to recover to, then don't add and recovery conditions. - return conditionFactories.stream() - .filter((ConditionFactory factory) -> factory.getType() != Condition.ConditionType.RECOVERY) - .map(ConditionFactory::build).collect(Collectors.toList()); - } - // The synchronizer can both fall back and recover. - return conditionFactories.stream().map(ConditionFactory::build).collect(Collectors.toList()); - } - /** * Runs the configured synchronizers, falling back / recovering as conditions allow, - * until the list is exhausted, the data source is closed, or a server-directed FDv1 - * fallback halts the data system. - * - * @return true when {@code runSynchronizers} halted in response to a server-directed - * FDv1 fallback directive that could not be satisfied (no FDv1 fallback - * synchronizer configured) -- the caller should NOT report exhaustion in - * that case because OFF has already been published with a specific error. + * until the data source is closed or a synchronizer reports that it shut down. */ - private boolean runSynchronizers() { + private void runSynchronizers() { // When runSynchronizers exists, no matter how it exits, the synchronizerStateManager will be closed. try { - Synchronizer synchronizer = sourceManager.getNextAvailableSynchronizerAndSetActive(); + Synchronizer synchronizer = buildNextAvailableSynchronizer(); // We want to continue running synchronizers for as long as any are available. while (synchronizer != null) { String synchronizerName = synchronizer.name(); logger.info("Synchronizer '{}' is starting.", synchronizerName); resetSynchronizerStatusDedupe(); + boolean parked = false; try { boolean running = true; + // Fallback and recovery apply only when another synchronizer can take over. + boolean canFallBack = sourceManager.getAvailableSynchronizerCount() > 1; + boolean canRecover = sourceManager.getAvailableSynchronizerCount() > 1 && !sourceManager.isPrimeSynchronizer(); + Instant recoveryTime = canRecover ? Instant.now().plus(recoveryTimeout) : null; + Instant fallbackTime = null; + // True once the loop has seen the deadline pass. + boolean recoveryMet = false; + boolean fallbackMet = false; + // The result the loop is waiting for, or null if it has not asked the synchronizer for one. + CompletableFuture nextResultFuture = null; + while (running) { + if (closed) { + return; + } + if (nextResultFuture == null) { + nextResultFuture = synchronizer.next(); + } - try (Conditions conditions = new Conditions(getConditions())) { - while (running) { - CompletableFuture nextResultFuture = synchronizer.next(); - - // The conditionsFuture will complete if any condition is met. Meeting any condition means we will - // switch to a different synchronizer. - Object res = CompletableFuture.anyOf(conditions.getFuture(), nextResultFuture).get(); - - if (res instanceof Condition) { - Condition c = (Condition) res; - switch (c.getType()) { - case FALLBACK: - logger.info( - "Fallback condition met, falling back from synchronizer '{}'.", - synchronizer.name() - ); - break; - case RECOVERY: - // For recovery, we will start at the first available synchronizer. - // So we reset the source index, and finding the source will start at the beginning. - sourceManager.resetSourceIndex(); - logger.info( - "Recovery condition met, moving from synchronizer '{}' to primary synchronizer.", - synchronizer.name() - ); - break; + // Wait for the next result, or for the next deadline that the loop has not seen pass. + Instant wakeTime = earliest( + recoveryMet ? null : recoveryTime, + fallbackMet ? null : fallbackTime, + sourceManager.getEarliestUnparkTime(), + sourceManager.getFDv1FallbackEndTime()); + if (awaitUntil(nextResultFuture, wakeTime)) { + try (FDv2SourceResult result = nextResultFuture.get()) { + // The pending result has arrived. + nextResultFuture = null; + + if (result.isResetThresholdMet()) { + sourceManager.resetCurrentSynchronizerParkCount(); } - // A running synchronizer will only have fallback and recovery conditions that it can act on. - // So, if there are no synchronizers to recover to or fallback to, then we will not have - // those conditions. - break; - } - - if (!(res instanceof FDv2SourceResult)) { - logger.error("Unexpected result type from synchronizer: {}", res.getClass().getName()); - continue; - } - - try (FDv2SourceResult result = (FDv2SourceResult) res) { - conditions.inform(result); switch (result.getResultType()) { case CHANGE_SET: // A data update breaks the "in a row" streak for status deduplication. resetSynchronizerStatusDedupe(); + // The synchronizer is healthy, so it no longer needs to fall back. + fallbackTime = null; + fallbackMet = false; dataSourceUpdates.apply(result.getChangeSet()); dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.VALID, null); // This could have been completed by any data source. But if it has not been completed before @@ -416,36 +403,51 @@ private boolean runSynchronizers() { switch (status.getState()) { case INTERRUPTED: maybeLogSynchronizerStatusChange( - synchronizer.name(), - status.getState() - ); - // Handled by conditions. + synchronizer.name(), + status.getState()); + // The fallback timeout runs from the first interruption. + if (canFallBack && fallbackTime == null) { + fallbackTime = Instant.now().plus(fallbackTimeout); + } dataSourceUpdates.updateStatus( - DataSourceStatusProvider.State.INTERRUPTED, - status.getErrorInfo()); + DataSourceStatusProvider.State.INTERRUPTED, + status.getErrorInfo()); break; case SHUTDOWN: maybeLogSynchronizerStatusChange( - synchronizer.name(), - status.getState() - ); - // We should be overall shutting down. - logger.debug("Synchronizer shutdown."); - return false; + synchronizer.name(), + status.getState()); + if (closed) { + // Closing the data source shut down the synchronizer. + logger.debug("Synchronizer shutdown."); + return; + } + // The synchronizer shut down by itself, so park it. + DataSourceStatusProvider.ErrorInfo shutdownError = + status.getErrorInfo() != null + ? status.getErrorInfo() + : new DataSourceStatusProvider.ErrorInfo( + DataSourceStatusProvider.ErrorKind.UNKNOWN, + 0, + "Synchronizer shut down unexpectedly", + Instant.now()); + parkActiveSynchronizer(synchronizer.name(), shutdownError); + parked = true; + running = false; + dataSourceUpdates.updateStatus( + DataSourceStatusProvider.State.INTERRUPTED, + shutdownError); + break; case TERMINAL_ERROR: maybeLogSynchronizerStatusChange( - synchronizer.name(), - status.getState() - ); - sourceManager.blockCurrentSynchronizer(); - logger.warn( - "Synchronizer '{}' permanently failed and will not be used again until application restart.", - synchronizer.name() - ); + synchronizer.name(), + status.getState()); + parkActiveSynchronizer(synchronizer.name(), status.getErrorInfo()); + parked = true; running = false; dataSourceUpdates.updateStatus( - DataSourceStatusProvider.State.INTERRUPTED, - status.getErrorInfo()); + DataSourceStatusProvider.State.INTERRUPTED, + status.getErrorInfo()); break; case GOODBYE: // We let the synchronizer handle this internally. @@ -453,58 +455,91 @@ private boolean runSynchronizers() { } break; } - // We have been requested to fall back to FDv1. Block every FDv2 - // synchronizer in one shot via fdv1Fallback() (which also unblocks the - // FDv1 fallback synchronizer, if one is configured). If FDv1 is - // configured we hand off to it; otherwise we halt the data system. - // An FDv1 fallback synchronizer asking to fall back again is ignored - // -- shouldn't happen in practice. - if (result.isFdv1Fallback() - && !sourceManager.isCurrentSynchronizerFDv1Fallback()) { - sourceManager.fdv1Fallback(); - if (sourceManager.hasFDv1Fallback()) { - logger.info("Falling back to an FDv1 fallback synchronizer."); - running = false; + if (result.isFdv1Fallback()) { + if (sourceManager.isCurrentSynchronizerFDv1Fallback()) { + // A directive during the FDv1 fallback replaces its TTL, and FDv1 keeps running. + Duration fallbackTTL = jittered(defaultFDv1FallbackTTL); + sourceManager.startFDv1Fallback(Instant.now().plus(fallbackTTL)); + logger.info("Retrying the configured synchronizers in {}s.", fallbackTTL.getSeconds()); } else { - // When the directive is signalled but no FDv1 fallback synchronizer - // is configured, halt the data system entirely. Surface OFF with - // the most recent error info (if any) and exit the synchronizer - // loop terminally. - logger.warn( - "Synchronizer '{}' requested FDv1 fallback, but no FDv1 fallback synchronizer is configured; halting the data system.", - synchronizer.name() - ); - DataSourceStatusProvider.ErrorInfo offError = - result.getStatus() != null ? result.getStatus().getErrorInfo() : null; - dataSourceUpdates.updateStatus( - DataSourceStatusProvider.State.OFF, - offError); - startFuture.complete(false); - return true; + // A directive to fall back to FDv1 ends the run of an FDv2 synchronizer. + fallBackToFDv1( + jittered(defaultFDv1FallbackTTL), + result.getStatus() != null ? result.getStatus().getErrorInfo() : null); + // The directive takes over, so there is no failover delay after a park. + parked = false; + running = false; } } } } + if (!running) { + break; + } + + // Check every deadline against the current time. + Instant now = Instant.now(); + sourceManager.unparkExpiredSynchronizers(); + if (sourceManager.endFDv1FallbackIfExpired()) { + logger.info("FDv1 fallback ended, returning to the configured synchronizers."); + break; + } + boolean recoveryJustMet = !recoveryMet && recoveryTime != null && !now.isBefore(recoveryTime); + boolean fallbackJustMet = !fallbackMet && fallbackTime != null && !now.isBefore(fallbackTime); + recoveryMet = recoveryMet || recoveryJustMet; + fallbackMet = fallbackMet || fallbackJustMet; + + // Recovery takes precedence over fallback. + if (recoveryMet && sourceManager.hasMorePreferredAvailableSynchronizer()) { + // For recovery, we will start at the first available synchronizer. + // So we reset the source index, and finding the source will start at the beginning. + sourceManager.resetSourceIndex(); + logger.info( + "Recovery condition met, moving from synchronizer '{}' to primary synchronizer.", + synchronizer.name()); + break; + } + if (fallbackMet && sourceManager.hasOtherAvailableSynchronizer()) { + logger.info( + "Fallback condition met, falling back from synchronizer '{}'.", + synchronizer.name()); + break; + } + + // Keep running this synchronizer. + if (fallbackJustMet) { + logger.info( + "Fallback condition met, but no other synchronizer is available; continuing with synchronizer '{}'.", + synchronizer.name()); + } + if (recoveryJustMet) { + logger.info( + "Recovery condition met, but no more-preferred synchronizer is available; continuing with synchronizer '{}'.", + synchronizer.name()); + } } - } catch (ExecutionException | InterruptedException | CancellationException e) { - dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.INTERRUPTED, - new DataSourceStatusProvider.ErrorInfo( + } catch (ExecutionException | InterruptedException | RuntimeException e) { + DataSourceStatusProvider.ErrorInfo errorInfo = new DataSourceStatusProvider.ErrorInfo( DataSourceStatusProvider.ErrorKind.UNKNOWN, 0, e.toString(), - new Date().toInstant() - )); + new Date().toInstant()); + dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.INTERRUPTED, errorInfo); Throwable root = e instanceof ExecutionException && e.getCause() != null ? e.getCause() : e; logger.error("Error running synchronizer '{}': {}", - synchronizer.name(), - root.getMessage() != null ? root.getMessage() : LogValues.exceptionSummary(root)); - // Move to the next synchronizer. + synchronizer.name(), + root.getMessage() != null ? root.getMessage() : LogValues.exceptionSummary(root)); + parkActiveSynchronizer(synchronizer.name(), errorInfo); + parked = true; + } + // The run of this synchronizer has ended, so stop it. + sourceManager.closeActiveSource(); + + if (parked && !waitUnlessClosed(jittered(failoverDelay))) { + break; } // Get the next available synchronizer and set it active - synchronizer = sourceManager.getNextAvailableSynchronizerAndSetActive(); - } - if (!closed) { - logger.warn("No more synchronizers available."); + synchronizer = buildNextAvailableSynchronizer(); } } catch (Exception e) { // We are not expecting to encounter this situation, but if we do, then we should log it. @@ -512,7 +547,6 @@ private boolean runSynchronizers() { } finally { sourceManager.close(); } - return false; } private static String detailForError(DataSourceStatusProvider.ErrorInfo errorInfo) { @@ -525,6 +559,188 @@ private static String detailForError(DataSourceStatusProvider.ErrorInfo errorInf return errorInfo.toString(); } + /** + * Waits for the given duration, or until the data source is closed. + * + * @param duration the time to wait + * @return true if the full duration passed, or false if the data source was closed + */ + private boolean waitUnlessClosed(Duration duration) { + try { + closeLatch.newFuture().get(Math.max(0, duration.toMillis()), TimeUnit.MILLISECONDS); + return false; + } catch (TimeoutException e) { + return true; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } catch (ExecutionException e) { + return false; + } + } + + /** + * Waits for the future to complete, until the given time, or until the data source is closed. + * + * @param future the future to wait for + * @param time the time to stop waiting, or null to wait until the future completes or the data source is closed + * @return true if the future completed + * @throws InterruptedException if the thread is interrupted while waiting + */ + private boolean awaitUntil(CompletableFuture future, Instant time) throws InterruptedException { + CompletableFuture futureOrClosed = CompletableFuture.anyOf(future, closeLatch.newFuture()); + try { + if (time == null) { + futureOrClosed.get(); + } else { + futureOrClosed.get( + Math.max(0, Duration.between(Instant.now(), time).toNanos()), TimeUnit.NANOSECONDS); + } + } catch (TimeoutException e) { + return false; + } catch (ExecutionException e) { + // The future completed with an error, which the caller gets from the future. + } + return future.isDone(); + } + + /** + * Gets the earliest of the given times. + * + * @param times the times, any of which may be null + * @return the earliest time, or null if every time is null + */ + private static Instant earliest(Instant... times) { + Instant earliest = null; + for (Instant time : times) { + if (time != null && (earliest == null || time.isBefore(earliest))) { + earliest = time; + } + } + return earliest; + } + + /** + * Subtracts a random jitter of up to half of the given delay. + * + * @param delay the delay before jitter + * @return the delay after jitter + */ + private static Duration jittered(Duration delay) { + return delay.minusMillis(ThreadLocalRandom.current().nextLong(delay.toMillis() / 2 + 1)); + } + + /** + * Computes the park delay for a synchronizer. + * + * @param parkCount the park count of the synchronizer, including the current failure + * @return the park delay + */ + private Duration parkDelay(int parkCount) { + double delayMillis = Math.min( + parkInitialDelay.toMillis() * Math.pow(2, parkCount - 1), + parkMaxDelay.toMillis()); + return jittered(Duration.ofMillis((long) delayMillis)); + } + + /** + * Parks the active synchronizer after an unexpected failure. + * + * @param synchronizerName the name of the active synchronizer, or null if it failed to build + * @param errorInfo the error that caused the park + */ + private void parkActiveSynchronizer(String synchronizerName, DataSourceStatusProvider.ErrorInfo errorInfo) { + Duration parkDelay = parkDelay(sourceManager.incrementCurrentSynchronizerParkCount()); + sourceManager.parkCurrentSynchronizer(Instant.now().plus(parkDelay)); + logger.warn( + "{} is backing off for {}s after an unexpected error ({}).", + synchronizerName != null ? "Synchronizer '" + synchronizerName + "'" + : "A synchronizer that failed to build", + parkDelay.getSeconds(), + detailForError(errorInfo)); + } + + /** + * Falls back to FDv1 for the given duration. If no FDv1 fallback synchronizer is configured, no synchronizer runs + * until the fallback ends. + * + * @param fallbackTTL how long the fallback lasts + * @param errorInfo the error that came with the directive, or null + */ + private void fallBackToFDv1(Duration fallbackTTL, DataSourceStatusProvider.ErrorInfo errorInfo) { + sourceManager.startFDv1Fallback(Instant.now().plus(fallbackTTL)); + if (sourceManager.hasFDv1Fallback()) { + logger.info("Falling back to FDv1 fallback synchronizer."); + } else { + logger.warn("Fallback to FDv1 was requested, but no FDv1 fallback synchronizer is configured."); + dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.INTERRUPTED, errorInfo); + } + logger.info("Retrying the configured synchronizers in {}s.", fallbackTTL.getSeconds()); + } + + /** + * Gets the next available synchronizer, builds it, and sets it active. If the FDv1 fallback has expired, this + * first makes the FDv2 synchronizers available again. While no synchronizer is available, this waits for the + * earliest unpark time or the end of the FDv1 fallback, whichever comes first. A synchronizer that fails to build + * is parked, and the next available synchronizer is tried after the failover delay. + * + * @return the active synchronizer, or null if the data source was closed + */ + private Synchronizer buildNextAvailableSynchronizer() { + while (!closed) { + if (sourceManager.endFDv1FallbackIfExpired()) { + logger.info("FDv1 fallback ended, returning to the configured synchronizers."); + } + Synchronizer synchronizer; + try { + synchronizer = sourceManager.getNextAvailableSynchronizerAndSetActive(); + } catch (RuntimeException e) { + DataSourceStatusProvider.ErrorInfo errorInfo = new DataSourceStatusProvider.ErrorInfo( + DataSourceStatusProvider.ErrorKind.UNKNOWN, + 0, + e.toString(), + Instant.now()); + dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.INTERRUPTED, errorInfo); + parkActiveSynchronizer(null, errorInfo); + if (!waitUnlessClosed(jittered(failoverDelay))) { + return null; + } + continue; + } + if (synchronizer != null) { + return synchronizer; + } + + Instant earliestUnparkTime = sourceManager.getEarliestUnparkTime(); + Instant fallbackEndTime = sourceManager.getFDv1FallbackEndTime(); + Duration wait; + if (earliestUnparkTime == null && fallbackEndTime == null) { + // This should never happen. + wait = parkInitialDelay; + logger.error("No synchronizer can be started; checking again in {}s.", wait.getSeconds()); + } else if (earliestUnparkTime == null) { + wait = Duration.between(Instant.now(), fallbackEndTime); + logger.warn("No synchronizer is available until the FDv1 fallback ends in {}s.", + Math.max(0, wait.getSeconds())); + } else if (fallbackEndTime == null) { + wait = Duration.between(Instant.now(), earliestUnparkTime); + logger.warn("All synchronizers are backing off for at least {}s.", Math.max(0, wait.getSeconds())); + } else { + Instant earliest = fallbackEndTime.isBefore(earliestUnparkTime) ? fallbackEndTime : earliestUnparkTime; + wait = Duration.between(Instant.now(), earliest); + logger.warn("All synchronizers are backing off for at least {}s.", Math.max(0, wait.getSeconds())); + } + // No synchronizer is running until the wait ends. + dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.INTERRUPTED, null); + if (!waitUnlessClosed(wait)) { + return null; + } + // Start the most-preferred synchronizer that is no longer parked. + sourceManager.resetSourceIndex(); + } + return null; + } + private void resetSynchronizerStatusDedupe() { lastLoggedSynchronizerDedupeName = null; lastLoggedSynchronizerDedupeStatus = null; @@ -535,8 +751,8 @@ private void maybeLogSynchronizerStatusChange(String sourceName, SourceSignal st return; } if (sourceName != null - && sourceName.equals(lastLoggedSynchronizerDedupeName) - && state == lastLoggedSynchronizerDedupeStatus) { + && sourceName.equals(lastLoggedSynchronizerDedupeName) + && state == lastLoggedSynchronizerDedupeStatus) { return; } lastLoggedSynchronizerDedupeName = sourceName; @@ -564,12 +780,22 @@ public boolean isInitialized() { @Override public void close() { closed = true; - // If there is an active source, we will shut it down, and that will result in the loop handling that source - // exiting. + closeLatch.release(); + // If there is an active source, we will shut it down, and that will make the loop handling that source exit. // If we do not have an active source, then the loop will check isShutdown when attempting to set one. When // it detects shutdown, it will exit the loop. sourceManager.close(); + // Wait for the data source to stop, so that it does not report a status or apply data after OFF. + Thread thread = runThread; + if (thread != null && thread != Thread.currentThread()) { + try { + thread.join(CLOSE_TIMEOUT.toMillis()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + dataSourceUpdates.updateStatus(DataSourceStatusProvider.State.OFF, null); // If this is already set, then this has no impact. @@ -577,132 +803,16 @@ public void close() { } private void maybeReportUnexpectedExhaustion(String message) { - if(!closed) { + if (!closed) { dataSourceUpdates.updateStatus( - DataSourceStatusProvider.State.OFF, - // If the data source was closed, then we just report we are OFF without an - // associated error. - new DataSourceStatusProvider.ErrorInfo( - DataSourceStatusProvider.ErrorKind.UNKNOWN, - 0, - message, - new Date().toInstant()) - ); - } - } - - /** - * Helper class to manage the lifecycle of conditions with automatic cleanup. - * - *

Before the aggregate completes, {@link #getFuture()} returns a - * fresh {@link CompletableFuture} per call. This matters because - * the run loop calls {@code CompletableFuture.anyOf(getFuture(), - * synchronizerNext)} on every iteration: if {@code getFuture()} returned - * the shared underlying aggregate while it was still pending, each - * {@code anyOf} call would permanently attach an {@code OrRelay} - * {@code Completion} to its {@code stack}. On a healthy primary - * synchronizer that streams ChangeSets without ever arming the fallback - * timer, the aggregate never completes, so those Completion nodes would - * accumulate monotonically for the synchronizer's full tenure -- a real - * memory leak proportional to event rate. - * - *

After the aggregate completes, {@link #getFuture()} returns the - * aggregate directly: any continuation registered on an already-completed - * CompletableFuture fires synchronously at registration time and is - * removed from the stack immediately by {@code cleanStack}, so the same - * accumulation cannot happen. - * - *

Fresh pre-completion futures are tracked in a {@link WeakHashMap}-backed - * set, so a fresh future whose only strong references were in the caller's - * loop iteration becomes garbage-collectable -- and automatically removed - * from {@code pending} -- once that iteration ends. - * - *

Package-private (rather than private) so that direct unit tests can - * exercise the API surface and assert per-call distinctness. - */ - static class Conditions implements AutoCloseable { - private final List conditions; - private final CompletableFuture aggregate; - private final Object lock = new Object(); - - /** - * Tracks futures previously returned by {@link #getFuture()} that have - * not yet been completed. Held weakly via {@link WeakHashMap} so that - * fresh futures abandoned by the caller (the typical end-of-iteration - * case) become GC-collectable. Set to {@code null} once the aggregate - * has fired and the entries have been drained. Mutated only under - * {@code lock}. - */ - private Set> pending = - Collections.newSetFromMap(new WeakHashMap<>()); - - public Conditions(List conditions) { - this.conditions = conditions; - this.aggregate = conditions.isEmpty() - ? new CompletableFuture<>() // Never completes if no conditions - : CompletableFuture.anyOf( - conditions.stream().map(Condition::execute).toArray(CompletableFuture[]::new)); - - // Single permanent listener. This is the only Completion node ever - // attached to aggregate.stack while the aggregate is still pending - // -- subsequent pre-completion getFuture() calls do not touch the - // aggregate at all. - this.aggregate.whenComplete((result, throwable) -> { - List> snapshot; - synchronized (lock) { - if (pending == null) { - return; - } - // Copy under the lock: the ArrayList holds strong - // references so entries that survived GC to this point - // stay alive until we complete them below. - snapshot = new ArrayList<>(pending); - pending = null; - } - for (CompletableFuture cf : snapshot) { - if (throwable != null) { - cf.completeExceptionally(throwable); - } else { - cf.complete(result); - } - } - }); - } - - /** - * Returns a future that will complete when the underlying aggregate - * condition fires. Pre-completion, this is a fresh future per call; - * post-completion, this is the aggregate itself (already done). - */ - public CompletableFuture getFuture() { - if (aggregate.isDone()) { - return aggregate; - } - - CompletableFuture fresh = new CompletableFuture<>(); - synchronized (lock) { - if (pending == null) { - // Raced with aggregate completion between isDone() and - // the lock acquisition; aggregate is now done. - return aggregate; - } - pending.add(fresh); - } - return fresh; - } - - public void inform(FDv2SourceResult result) { - conditions.forEach(c -> c.inform(result)); - } - - @Override - public void close() { - conditions.forEach(Condition::close); - synchronized (lock) { - if (pending != null) { - pending.clear(); - } - } + DataSourceStatusProvider.State.OFF, + // If the data source was closed, then we just report we are OFF without an + // associated error. + new DataSourceStatusProvider.ErrorInfo( + DataSourceStatusProvider.ErrorKind.UNKNOWN, + 0, + message, + new Date().toInstant())); } } } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java deleted file mode 100644 index 30f81b28..00000000 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSourceConditions.java +++ /dev/null @@ -1,183 +0,0 @@ -package com.launchdarkly.sdk.server; - -import com.launchdarkly.sdk.fdv2.SourceResultType; -import com.launchdarkly.sdk.fdv2.SourceSignal; -import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; - -import java.io.Closeable; -import java.io.IOException; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -/** - * Container class for FDv2 data source conditions and related types. - *

- * This class is non-constructable and serves only as a namespace for condition-related types. - * Package-private for internal use and testing. - */ -class FDv2DataSourceConditions { - /** - * Private constructor to prevent instantiation. - */ - private FDv2DataSourceConditions() { - } - - /** - * Package-private for testing. - */ - interface Condition extends Closeable { - enum ConditionType { - FALLBACK, - RECOVERY, - } - - CompletableFuture execute(); - - void inform(FDv2SourceResult sourceResult); - - void close(); - - ConditionType getType(); - } - - interface ConditionFactory { - Condition build(); - - Condition.ConditionType getType(); - } - - static abstract class TimedCondition implements Condition { - protected final CompletableFuture resultFuture = new CompletableFuture<>(); - - protected final ScheduledExecutorService sharedExecutor; - - /** - * Future for the timeout task, if any. Will be null when no timeout is active. - */ - protected ScheduledFuture timerFuture; - - /** - * Timeout duration for the fallback operation. - */ - protected final long timeoutSeconds; - - public TimedCondition(ScheduledExecutorService sharedExecutor, long timeoutSeconds) { - this.sharedExecutor = sharedExecutor; - this.timeoutSeconds = timeoutSeconds; - } - - @Override - public CompletableFuture execute() { - return resultFuture; - } - - @Override - public void close() { - if (timerFuture != null) { - timerFuture.cancel(false); - timerFuture = null; - } - } - - static abstract class Factory implements ConditionFactory { - protected final ScheduledExecutorService sharedExecutor; - protected final long timeoutSeconds; - - public Factory(ScheduledExecutorService sharedExecutor, long timeout) { - this.sharedExecutor = sharedExecutor; - this.timeoutSeconds = timeout; - } - } - } - - /** - * This condition is used to determine if a fallback should be performed. It is informed of each data source result - * via {@link #inform(FDv2SourceResult)}. Based on the results, it updates its internal state. When the fallback - * condition is met, then the {@link java.util.concurrent.Future} returned by {@link #execute()} will complete. - *

- * This is package-private, instead of private, for ease of testing. - */ - static class FallbackCondition extends TimedCondition { - static class Factory extends TimedCondition.Factory { - public Factory(ScheduledExecutorService sharedExecutor, long timeout) { - super(sharedExecutor, timeout); - } - - @Override - public Condition build() { - return new FallbackCondition(sharedExecutor, timeoutSeconds); - } - - @Override - public ConditionType getType() { - return ConditionType.FALLBACK; - } - } - - public FallbackCondition(ScheduledExecutorService sharedExecutor, long timeoutSeconds) { - super(sharedExecutor, timeoutSeconds); - } - - @Override - public void inform(FDv2SourceResult sourceResult) { - if (sourceResult.getResultType() == SourceResultType.CHANGE_SET) { - if (timerFuture != null) { - timerFuture.cancel(false); - timerFuture = null; - } - } - if (sourceResult.getResultType() == SourceResultType.STATUS && sourceResult.getStatus().getState() == SourceSignal.INTERRUPTED) { - if (timerFuture == null) { - timerFuture = sharedExecutor.schedule(() -> { - resultFuture.complete(this); - return null; - }, timeoutSeconds, TimeUnit.SECONDS); - } - } - } - - @Override - public ConditionType getType() { - return ConditionType.FALLBACK; - } - } - - static class RecoveryCondition extends TimedCondition { - - static class Factory extends TimedCondition.Factory { - public Factory(ScheduledExecutorService sharedExecutor, long timeout) { - super(sharedExecutor, timeout); - } - - @Override - public Condition build() { - return new RecoveryCondition(sharedExecutor, timeoutSeconds); - } - - @Override - public ConditionType getType() { - return ConditionType.RECOVERY; - } - } - - public RecoveryCondition(ScheduledExecutorService sharedExecutor, long timeoutSeconds) { - super(sharedExecutor, timeoutSeconds); - timerFuture = sharedExecutor.schedule(() -> { - resultFuture.complete(this); - return null; - }, timeoutSeconds, TimeUnit.SECONDS); - } - - @Override - public void inform(FDv2SourceResult sourceResult) { - // Time-based recovery. - } - - @Override - public ConditionType getType() { - return ConditionType.RECOVERY; - } - } -} diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java index 5053625b..2e44afba 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/FDv2DataSystem.java @@ -176,8 +176,7 @@ static FDv2DataSystem create( fdv1FallbackFactory, dataSourceUpdates, config.threadPriority, - clientContext.getBaseLogger().subLogger(Loggers.DATA_SOURCE_LOGGER_NAME), - clientContext.sharedExecutor + clientContext.getBaseLogger().subLogger(Loggers.DATA_SOURCE_LOGGER_NAME) ); } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingBase.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingBase.java index c21bd5ba..cb429c60 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingBase.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingBase.java @@ -1,6 +1,7 @@ package com.launchdarkly.sdk.server; import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.sdk.internal.http.FailureClass; import com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event; import com.launchdarkly.sdk.internal.fdv2.sources.FDv2ProtocolHandler; import com.launchdarkly.sdk.fdv2.Selector; @@ -65,7 +66,10 @@ protected CompletableFuture poll(Selector selector, boolean on e.toString(), new Date().toInstant() ); - return oneShot ? FDv2SourceResult.terminalError(info, fdv1Fallback) : FDv2SourceResult.interrupted(info, fdv1Fallback); + boolean recoverable = classifyTransportFailure(e) == FailureClass.NORMAL; + return oneShot || !recoverable + ? FDv2SourceResult.terminalError(info, fdv1Fallback) + : FDv2SourceResult.interrupted(info, fdv1Fallback); } else if (ex instanceof SerializationException) { SerializationException e = (SerializationException) ex; logger.error("Polling request received malformed data: {}", e.toString()); @@ -101,7 +105,7 @@ protected CompletableFuture poll(Selector selector, boolean on } if(!pollingResponse.isSuccess()) { int statusCode = pollingResponse.getStatusCode(); - boolean recoverable = statusCode <= 0 || isHttpErrorRecoverable(statusCode); + boolean recoverable = classifyHttpFailure(statusCode) == FailureClass.NORMAL; DataSourceStatusProvider.ErrorInfo errorInfo = DataSourceStatusProvider.ErrorInfo.fromHttpError(statusCode); logger.error("Polling request failed with HTTP error: {}", statusCode); // For a one-shot request all errors are terminal. diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingSynchronizerImpl.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingSynchronizerImpl.java index 7d0f8292..bf7535dd 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingSynchronizerImpl.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/PollingSynchronizerImpl.java @@ -10,6 +10,11 @@ import java.util.concurrent.*; class PollingSynchronizerImpl extends PollingBase implements Synchronizer { + /** + * The number of consecutive successful polls that meets the reset threshold. + */ + private static final int RESET_THRESHOLD_SUCCESSFUL_POLLS = 2; + private final CompletableFuture shutdownFuture = new CompletableFuture<>(); private final SelectorSource selectorSource; @@ -17,6 +22,12 @@ class PollingSynchronizerImpl extends PollingBase implements Synchronizer { private final IterableAsyncQueue resultQueue = new IterableAsyncQueue<>(); + /** + * Only {@link #doPoll()} reads or writes this. {@code scheduleAtFixedRate} never starts a run of + * {@code doPoll} before the previous run has finished. + */ + private int consecutiveSuccessfulPolls = 0; + public PollingSynchronizerImpl( FDv2Requestor requestor, LDLogger logger, @@ -47,10 +58,15 @@ private void doPoll() { boolean shouldShutdown = false; switch (res.getResultType()) { case CHANGE_SET: + consecutiveSuccessfulPolls++; + if (consecutiveSuccessfulPolls == RESET_THRESHOLD_SUCCESSFUL_POLLS) { + res = res.withResetThresholdMet(); + } break; case STATUS: switch (res.getStatus().getState()) { case INTERRUPTED: + consecutiveSuccessfulPolls = 0; break; case SHUTDOWN: // The base poller doesn't emit shutdown, we instead handle it at this level. diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java index dd093d9a..e88960ce 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SourceManager.java @@ -4,6 +4,7 @@ import java.io.Closeable; import java.io.IOException; +import java.time.Instant; import java.util.List; /** @@ -36,6 +37,12 @@ class SourceManager implements Closeable { */ private SynchronizerFactoryWithState currentSynchronizerFactory; + /** + * The time at which the fallback to FDv1 ends, or null if the SDK is not falling back to FDv1. Accessed only + * under activeSourceLock. + */ + private Instant fdv1FallbackEndTime; + public SourceManager(List synchronizers, List> initializers) { this.synchronizers = synchronizers; this.initializers = initializers; @@ -60,16 +67,77 @@ public boolean hasFDv1Fallback() { return false; } + public boolean hasFDv2Synchronizers() { + for (SynchronizerFactoryWithState factory : synchronizers) { + if (!factory.isFDv1Fallback()) { + return true; + } + } + return false; + } + + /** + * Block all synchronizers aside from the fdv1 fallback and unblock the fdv1 fallback, until the given time. + * + * @param endTime the time at which the fallback to FDv1 ends + */ + public void startFDv1Fallback(Instant endTime) { + synchronized (activeSourceLock) { + fdv1FallbackEndTime = endTime; + for (SynchronizerFactoryWithState factory : synchronizers) { + if (factory.isFDv1Fallback()) { + factory.unblock(); + } else { + factory.block(); + } + } + } + } + /** - * Block all synchronizers aside from the fdv1 fallback and unblock the fdv1 fallback. + * Stops parking every synchronizer whose unpark time has arrived. */ - public void fdv1Fallback() { - for (SynchronizerFactoryWithState factory : synchronizers) { - if(factory.isFDv1Fallback()) { - factory.unblock(); - } else { - factory.block(); + public void unparkExpiredSynchronizers() { + synchronized (activeSourceLock) { + Instant now = Instant.now(); + for (SynchronizerFactoryWithState factory : synchronizers) { + factory.unparkIfExpired(now); + } + } + } + + /** + * The time at which the fallback to FDv1 ends. + * + * @return the end time, or null if the SDK is not falling back to FDv1 + */ + public Instant getFDv1FallbackEndTime() { + synchronized (activeSourceLock) { + return fdv1FallbackEndTime; + } + } + + /** + * Ends the fallback to FDv1 if its end time has arrived. This unblocks the FDv2 synchronizers, blocks the fdv1 + * fallback, and starts the search for the next synchronizer from the most-preferred one. + * + * @return true if the fallback ended + */ + public boolean endFDv1FallbackIfExpired() { + synchronized (activeSourceLock) { + if (fdv1FallbackEndTime == null || Instant.now().isBefore(fdv1FallbackEndTime)) { + return false; } + fdv1FallbackEndTime = null; + for (SynchronizerFactoryWithState factory : synchronizers) { + if (factory.isFDv1Fallback()) { + factory.block(); + } else { + factory.unblock(); + } + } + synchronizerIndex = -1; + return true; } } @@ -88,6 +156,7 @@ public void fdv1Fallback() { */ private SynchronizerFactoryWithState getNextAvailableSynchronizer() { SynchronizerFactoryWithState factory = null; + unparkExpiredSynchronizers(); int visited = 0; while(visited < synchronizers.size()) { @@ -101,7 +170,7 @@ private SynchronizerFactoryWithState getNextAvailableSynchronizer() { } SynchronizerFactoryWithState candidate = synchronizers.get(synchronizerIndex); - if (candidate.getState() == SynchronizerFactoryWithState.State.Available) { + if (candidate.isEligible()) { factory = candidate; break; } @@ -116,9 +185,6 @@ private SynchronizerFactoryWithState getNextAvailableSynchronizer() { *

* If shutdown has been initiated, returns null without building or setting a source. * Any previously active source will be closed before setting the new one. - *

- * The current synchronizer factory can be retrieved with {@link #blockCurrentSynchronizer()} - * or {@link #isCurrentSynchronizerFDv1Fallback()} to interact with it. * * @return the built synchronizer that is now active, or null if no more synchronizers are available or shutdown has been initiated */ @@ -176,11 +242,46 @@ private FDv2DataSource.DataSourceFactory getNextInitializer() { return initializers.get(initializerIndex); } - public void blockCurrentSynchronizer() { + /** + * Records an unexpected failure of the current synchronizer. + * + * @return the park count of the current synchronizer, including this failure + */ + public int incrementCurrentSynchronizerParkCount() { + synchronized (activeSourceLock) { + return currentSynchronizerFactory.incrementParkCount(); + } + } + + public void resetCurrentSynchronizerParkCount() { + synchronized (activeSourceLock) { + currentSynchronizerFactory.resetParkCount(); + } + } + + public void parkCurrentSynchronizer(Instant unparkTime) { + synchronized (activeSourceLock) { + currentSynchronizerFactory.park(unparkTime); + } + } + + /** + * The earliest unpark time of the available synchronizers. + * + * @return the earliest unpark time, or null if no available synchronizer is parked + */ + public Instant getEarliestUnparkTime() { synchronized (activeSourceLock) { - if (currentSynchronizerFactory != null) { - currentSynchronizerFactory.block(); + Instant earliest = null; + for (SynchronizerFactoryWithState factory : synchronizers) { + Instant unparkTime = factory.getUnparkTime(); + if (factory.getState() == SynchronizerFactoryWithState.State.Available + && unparkTime != null + && (earliest == null || unparkTime.isBefore(earliest))) { + earliest = unparkTime; + } } + return earliest; } } @@ -223,6 +324,41 @@ public Initializer getNextInitializerAndSetActive() { } } + /** + * True if a synchronizer other than the current one is available. + * + * @return true if another synchronizer is available + */ + public boolean hasOtherAvailableSynchronizer() { + synchronized (activeSourceLock) { + for (SynchronizerFactoryWithState factory : synchronizers) { + if (factory != currentSynchronizerFactory && factory.isEligible()) { + return true; + } + } + return false; + } + } + + /** + * True if a synchronizer earlier in the list than the current one is available. + * + * @return true if a more-preferred synchronizer is available + */ + public boolean hasMorePreferredAvailableSynchronizer() { + synchronized (activeSourceLock) { + for (SynchronizerFactoryWithState factory : synchronizers) { + if (factory == currentSynchronizerFactory) { + return false; + } + if (factory.isEligible()) { + return true; + } + } + return false; + } + } + /** * Determine if the currently active synchronizer is the prime (first available) synchronizer. * @return true if the current synchronizer is the prime synchronizer, false otherwise @@ -260,6 +396,19 @@ public int getAvailableSynchronizerCount() { } + /** + * Closes the active source, if there is one. + */ + public void closeActiveSource() { + synchronized (activeSourceLock) { + Closeable source = activeSource; + activeSource = null; + if (source != null) { + safeClose(source); + } + } + } + /** * Close the state manager and shut down any active source. * Implements AutoCloseable to enable try-with-resources usage. @@ -268,15 +417,7 @@ public int getAvailableSynchronizerCount() { public void close() { synchronized (activeSourceLock) { isShutdown = true; - if (activeSource != null) { - try { - activeSource.close(); - } catch (IOException e) { - // We are done with this synchronizer, so we don't care if it encounters - // an error condition. - } - activeSource = null; - } + closeActiveSource(); } } @@ -290,7 +431,7 @@ private void safeClose(Closeable closeable) { } try { closeable.close(); - } catch (IOException e) { + } catch (IOException | RuntimeException e) { // Ignore close exceptions. } } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java index 5babb2d4..d2c2847c 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/StreamingSynchronizerImpl.java @@ -18,6 +18,8 @@ import com.launchdarkly.sdk.internal.fdv2.payloads.FDv2Event; import com.launchdarkly.sdk.internal.fdv2.sources.FDv2ProtocolHandler; import com.launchdarkly.sdk.fdv2.Selector; +import com.launchdarkly.sdk.internal.http.FailureClass; +import com.launchdarkly.sdk.internal.http.HttpErrors; import com.launchdarkly.sdk.internal.http.HttpHelpers; import com.launchdarkly.sdk.internal.http.HttpProperties; import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; @@ -25,6 +27,7 @@ import com.launchdarkly.sdk.server.datasources.Synchronizer; import com.launchdarkly.sdk.server.interfaces.DataSourceStatusProvider; import com.launchdarkly.sdk.fdv2.ChangeSet; +import com.launchdarkly.sdk.fdv2.ChangeSetType; import com.launchdarkly.sdk.server.subsystems.DataStoreTypes; import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; @@ -36,17 +39,18 @@ import org.jetbrains.annotations.NotNull; import java.io.Reader; +import java.util.Collections; import java.util.Map; import java.net.URI; import java.time.Duration; import java.time.Instant; import java.util.Objects; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import static com.launchdarkly.sdk.internal.http.HttpErrors.checkIfErrorIsRecoverableAndLog; - /** * Implementation of FDv2 streaming synchronizer. * Maintains a long-running streaming connection and queues results as they arrive. @@ -54,6 +58,11 @@ class StreamingSynchronizerImpl implements Synchronizer { private static final Duration DEAD_CONNECTION_INTERVAL = Duration.ofSeconds(300); + /** + * The time a stream must stay healthy, from its first valid result, to meet the reset threshold. + */ + static final Duration RESET_THRESHOLD = Duration.ofSeconds(60); + private final HttpProperties httpProperties; private final SelectorSource selectorSource; final URI streamUri; @@ -71,16 +80,28 @@ class StreamingSynchronizerImpl implements Synchronizer { private final int threadPriority; private final DiagnosticStore diagnosticStore; + private final ScheduledExecutorService sharedExecutor; + private final Duration resetThreshold; private volatile long streamStarted = 0; + private final Object resetThresholdLock = new Object(); + + /** + * The timer that reports that the current stream met its reset threshold, or null if the current stream has + * not produced a valid result. Guarded by {@link #resetThresholdLock}. + */ + private ScheduledFuture resetThresholdFuture; + public StreamingSynchronizerImpl( HttpProperties httpProperties, URI baseUri, String requestPath, LDLogger logger, SelectorSource selectorSource, + ScheduledExecutorService sharedExecutor, String payloadFilter, Duration initialReconnectDelaySeconds, + Duration resetThreshold, int threadPriority, DiagnosticStore diagnosticStore ) { @@ -92,6 +113,8 @@ public StreamingSynchronizerImpl( this.initialReconnectDelay = initialReconnectDelaySeconds; this.threadPriority = threadPriority; this.diagnosticStore = diagnosticStore; + this.sharedExecutor = sharedExecutor; + this.resetThreshold = resetThreshold; // The stream will lazily start when `next` is called. } @@ -187,6 +210,7 @@ private Thread getRunThread() { logger.error("Stream thread ended with exception: {}", LogValues.exceptionSummary(e)); logger.debug(LogValues.exceptionTrace(e)); + cancelResetThresholdTimer(); recordStreamInit(true); // Record failed init for unexpected thread exception DataSourceStatusProvider.ErrorInfo errorInfo = new DataSourceStatusProvider.ErrorInfo( @@ -195,9 +219,8 @@ private Thread getRunThread() { e.toString(), Instant.now() ); - // We aren't restarting the event source here. We aren't going to automatically recover, so the - // data system will move to the next source when it determined this source is unhealthy. - resultQueue.put(FDv2SourceResult.interrupted(errorInfo, getFallback(e))); + // This synchronizer cannot recover, so the failure is terminal. The data system will start a new one. + shutdownFuture.complete(FDv2SourceResult.terminalError(errorInfo, getFallback(e))); } finally { eventSource.close(); } @@ -238,6 +261,7 @@ public void close() { } } + cancelResetThresholdTimer(); shutdownFuture.complete(FDv2SourceResult.shutdown()); } @@ -248,8 +272,58 @@ private void recordStreamInit(boolean failed) { } } + /** + * Starts the reset threshold timer if the current stream does not have one yet. Call this for each + * valid result, so that the first valid result on a stream starts the timer. + */ + private void startResetThresholdTimerIfNeeded() { + synchronized (resetThresholdLock) { + if (resetThresholdFuture != null) { + return; + } + resetThresholdFuture = sharedExecutor.schedule( + this::reportResetThresholdMet, + resetThreshold.toMillis(), + TimeUnit.MILLISECONDS); + } + } + + /** + * Stops the reset threshold timer of the current stream. Call this when the stream ends. + */ + private void cancelResetThresholdTimer() { + synchronized (resetThresholdLock) { + if (resetThresholdFuture != null) { + resetThresholdFuture.cancel(false); + resetThresholdFuture = null; + } + } + } + + private void reportResetThresholdMet() { + synchronized (resetThresholdLock) { + // The timer can start to run before a cancel. Do not report anything for a stream that ended. + if (resetThresholdFuture == null) { + return; + } + ChangeSet>>> noChanges = new ChangeSet<>( + ChangeSetType.None, + Selector.EMPTY, + Collections.emptyList(), + /* environmentId= */ null, + /* shouldPersist= */ true); + resultQueue.put(FDv2SourceResult.changeSet(noChanges, /* fdv1Fallback= */ false).withResetThresholdMet()); + } + } + private boolean handleEvent(StreamEvent event) { if (event instanceof MessageEvent) { + synchronized (closeLock) { + // The event source can connect again after close(). Stop the stream instead of handling the event. + if (closed) { + return false; + } + } handleMessage((MessageEvent) event); return true; } else if (event instanceof FaultEvent) { @@ -296,6 +370,7 @@ private void handleMessage(MessageEvent event) { // Reset to 0 after successful init to prevent duplicate recordings until next restart streamStarted = 0; result = FDv2SourceResult.changeSet(converted, getFallback(event)); + startResetThresholdTimerIfNeeded(); } catch (Exception e) { logger.error("Failed to convert FDv2 changeset: {}", LogValues.exceptionSummary(e)); logger.debug(LogValues.exceptionTrace(e)); @@ -347,6 +422,9 @@ private void handleMessage(MessageEvent event) { result = FDv2SourceResult.interrupted(internalError, getFallback(event)); if(kind == DataSourceStatusProvider.ErrorKind.INVALID_DATA) { restartStream(true); + } else { + // The stream stays open, but the failure restarts the measurement of healthy operation. + cancelResetThresholdTimer(); } break; @@ -387,19 +465,26 @@ private boolean handleError(StreamException e) { return true; } + synchronized (closeLock) { + // The event source can connect again after close(). Stop the stream instead of reconnecting. + if (closed) { + return false; + } + } + + cancelResetThresholdTimer(); recordStreamInit(true); if (e instanceof StreamHttpErrorException) { int status = ((StreamHttpErrorException) e).getCode(); DataSourceStatusProvider.ErrorInfo errorInfo = DataSourceStatusProvider.ErrorInfo.fromHttpError(status); - boolean recoverable = checkIfErrorIsRecoverableAndLog(logger, - "HTTP error " + status, - "in FDv2 streaming connection", + FailureClass failureClass = HttpErrors.classifyAndLogHttpFailure(logger, status, + "in FDv2 streaming connection", "will retry"); - if (!recoverable) { + if (failureClass == FailureClass.UNEXPECTED) { shutdownFuture.complete(FDv2SourceResult.terminalError(errorInfo, getFallback(e))); return false; } else { @@ -419,6 +504,10 @@ private boolean handleError(StreamException e) { e.toString(), Instant.now() ); + if (HttpErrors.classifyTransportFailure(e) == FailureClass.UNEXPECTED) { + shutdownFuture.complete(FDv2SourceResult.terminalError(errorInfo, getFallback(e))); + return false; + } resultQueue.put(FDv2SourceResult.interrupted(errorInfo, getFallback(e))); streamStarted = System.currentTimeMillis(); return true; // allow reconnect @@ -426,6 +515,7 @@ private boolean handleError(StreamException e) { private void restartStream(boolean failed) { Objects.requireNonNull(eventSource, "eventSource must not be null"); + cancelResetThresholdTimer(); recordStreamInit(failed); streamStarted = System.currentTimeMillis(); eventSource.interrupt(); diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java index 31f02cc1..6ee885bd 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/SynchronizerFactoryWithState.java @@ -2,6 +2,8 @@ import com.launchdarkly.sdk.server.datasources.Synchronizer; +import java.time.Instant; + class SynchronizerFactoryWithState { public enum State { /** @@ -21,6 +23,16 @@ public enum State { private boolean isFDv1Fallback = false; + /** + * The number of unexpected failures since the synchronizer last reported that it met its reset threshold. + */ + private int parkCount = 0; + + /** + * The time at which the synchronizer stops being parked, or null if it is not parked. + */ + private Instant unparkTime = null; + public SynchronizerFactoryWithState(FDv2DataSource.DataSourceFactory factory) { this.factory = factory; @@ -51,4 +63,50 @@ public void unblock() { public boolean isFDv1Fallback() { return isFDv1Fallback; } + + /** + * Records an unexpected failure. + * + * @return the park count, including this failure + */ + public int incrementParkCount() { + return ++parkCount; + } + + public void resetParkCount() { + parkCount = 0; + } + + public void park(Instant unparkTime) { + this.unparkTime = unparkTime; + } + + /** + * The time at which the synchronizer stops being parked. + * + * @return the unpark time, or null if the synchronizer is not parked + */ + public Instant getUnparkTime() { + return unparkTime; + } + + /** + * Stops parking the synchronizer if its unpark time has arrived. + * + * @param now the current time + */ + public void unparkIfExpired(Instant now) { + if (unparkTime != null && !now.isBefore(unparkTime)) { + unparkTime = null; + } + } + + /** + * True if the synchronizer is available and is not parked. + * + * @return true if the synchronizer can be started + */ + public boolean isEligible() { + return state == State.Available && unparkTime == null; + } } diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/datasources/FDv2SourceResult.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/datasources/FDv2SourceResult.java index 32b6c72b..d25d77c1 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/datasources/FDv2SourceResult.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/datasources/FDv2SourceResult.java @@ -72,18 +72,21 @@ public String getReason() { private final Status status; private final SourceResultType resultType; private final boolean fdv1Fallback; + private final boolean resetThresholdMet; private final Function completionCallback; private FDv2SourceResult( ChangeSet>>> changeSet, Status status, SourceResultType resultType, boolean fdv1Fallback, + boolean resetThresholdMet, Function completionCallback ) { this.changeSet = changeSet; this.status = status; this.resultType = resultType; this.fdv1Fallback = fdv1Fallback; + this.resetThresholdMet = resetThresholdMet; this.completionCallback = completionCallback; } @@ -93,10 +96,11 @@ public static FDv2SourceResult interrupted(DataSourceStatusProvider.ErrorInfo er public static FDv2SourceResult interrupted(DataSourceStatusProvider.ErrorInfo errorInfo, boolean fdv1Fallback, Function completionCallback) { return new FDv2SourceResult( - null, + /* changeSet= */ null, Status.interrupted(errorInfo), SourceResultType.STATUS, fdv1Fallback, + /* resetThresholdMet= */ false, completionCallback); } @@ -105,10 +109,12 @@ public static FDv2SourceResult shutdown() { } public static FDv2SourceResult shutdown(Function completionCallback) { - return new FDv2SourceResult(null, + return new FDv2SourceResult( + /* changeSet= */ null, Status.shutdown(), SourceResultType.STATUS, - false, + /* fdv1Fallback= */ false, + /* resetThresholdMet= */ false, completionCallback); } @@ -117,10 +123,12 @@ public static FDv2SourceResult terminalError(DataSourceStatusProvider.ErrorInfo } public static FDv2SourceResult terminalError(DataSourceStatusProvider.ErrorInfo errorInfo, boolean fdv1Fallback, Function completionCallback) { - return new FDv2SourceResult(null, + return new FDv2SourceResult( + /* changeSet= */ null, Status.terminalError(errorInfo), SourceResultType.STATUS, fdv1Fallback, + /* resetThresholdMet= */ false, completionCallback); } @@ -131,9 +139,10 @@ public static FDv2SourceResult changeSet(ChangeSet>>> changeSet, boolean fdv1Fallback, Function completionCallback) { return new FDv2SourceResult( changeSet, - null, + /* status= */ null, SourceResultType.CHANGE_SET, fdv1Fallback, + /* resetThresholdMet= */ false, completionCallback); } @@ -143,10 +152,11 @@ public static FDv2SourceResult goodbye(String reason, boolean fdv1Fallback) { public static FDv2SourceResult goodbye(String reason, boolean fdv1Fallback, Function completionCallback) { return new FDv2SourceResult( - null, + /* changeSet= */ null, Status.goodbye(reason), SourceResultType.STATUS, fdv1Fallback, + /* resetThresholdMet= */ false, completionCallback); } @@ -166,6 +176,30 @@ public boolean isFdv1Fallback() { return fdv1Fallback; } + /** + * True if the source met its reset threshold at the time of this result. + * + * @return true if the source met its reset threshold + */ + public boolean isResetThresholdMet() { + return resetThresholdMet; + } + + /** + * Creates a copy of this result that reports that the source met its reset threshold. + * + * @return a new FDv2SourceResult with {@link #isResetThresholdMet()} set to true + */ + public FDv2SourceResult withResetThresholdMet() { + return new FDv2SourceResult( + changeSet, + status, + resultType, + fdv1Fallback, + /* resetThresholdMet= */ true, + completionCallback); + } + /** * Creates a new result wrapping this one with an additional completion callback. *

@@ -183,7 +217,13 @@ public FDv2SourceResult withCompletion(Function newCallback) { } return null; }; - return new FDv2SourceResult(changeSet, status, resultType, fdv1Fallback, combinedCallback); + return new FDv2SourceResult( + changeSet, + status, + resultType, + fdv1Fallback, + resetThresholdMet, + combinedCallback); } @Override diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapterTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapterTest.java index ce81c226..bbdbc8e3 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapterTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/DataSourceSynchronizerAdapterTest.java @@ -4,6 +4,7 @@ import com.launchdarkly.sdk.fdv2.SourceResultType; import com.launchdarkly.sdk.fdv2.SourceSignal; import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; +import com.launchdarkly.sdk.server.interfaces.DataSourceStatusProvider; import com.launchdarkly.sdk.server.subsystems.DataSource; import com.launchdarkly.sdk.server.DataStoreTestTypes.DataBuilder; import com.launchdarkly.sdk.server.subsystems.DataSourceUpdateSink; @@ -361,4 +362,105 @@ public void environmentIdIsRetainedForSubsequentChangeSets() throws Exception { adapter.close(); } + + /** + * Test that an interrupted status with an unexpected HTTP error is reported as terminal, and an + * interrupted status with a normal HTTP error is reported as interrupted. + */ + @Test + public void interruptedStatusIsTerminalOnlyForUnexpectedHttpErrors() throws Exception { + AtomicReference capturedSink = new AtomicReference<>(); + + DataSourceSynchronizerAdapter adapter = new DataSourceSynchronizerAdapter(sink -> { + capturedSink.set(sink); + return new MockDataSource(null, null); + }); + resourcesToClose.add(adapter); + + // Report a 503 as interrupted, as the FDv1 data sources do. + CompletableFuture normalFuture = adapter.next(); + capturedSink.get().updateStatus(DataSourceStatusProvider.State.INTERRUPTED, + DataSourceStatusProvider.ErrorInfo.fromHttpError(503)); + FDv2SourceResult normalResult = normalFuture.get(2, TimeUnit.SECONDS); + + // The 503 stays interrupted. + assertEquals(SourceSignal.INTERRUPTED, normalResult.getStatus().getState()); + + // Report a 401 as interrupted, as the FDv1 data sources do. + CompletableFuture unexpectedFuture = adapter.next(); + capturedSink.get().updateStatus(DataSourceStatusProvider.State.INTERRUPTED, + DataSourceStatusProvider.ErrorInfo.fromHttpError(401)); + FDv2SourceResult unexpectedResult = unexpectedFuture.get(2, TimeUnit.SECONDS); + + // The 401 becomes terminal. + assertEquals(SourceSignal.TERMINAL_ERROR, unexpectedResult.getStatus().getState()); + assertEquals(401, unexpectedResult.getStatus().getErrorInfo().getStatusCode()); + + adapter.close(); + } + + /** Test that an OFF status with no error is reported as a shutdown. */ + @Test + public void offStatusWithoutErrorIsReportedAsShutdown() throws Exception { + AtomicReference capturedSink = new AtomicReference<>(); + + DataSourceSynchronizerAdapter adapter = new DataSourceSynchronizerAdapter(sink -> { + capturedSink.set(sink); + return new MockDataSource(null, null); + }); + resourcesToClose.add(adapter); + + // Report OFF with no error. + CompletableFuture resultFuture = adapter.next(); + capturedSink.get().updateStatus(DataSourceStatusProvider.State.OFF, null); + FDv2SourceResult result = resultFuture.get(2, TimeUnit.SECONDS); + + // The adapter reports that the data source shut down. + assertEquals(SourceSignal.SHUTDOWN, result.getStatus().getState()); + + adapter.close(); + } + + /** Test that two VALID statuses in a row meet the reset threshold, and a failure restarts the count. */ + @Test + public void resetThresholdIsMetAfterTwoValidStatusesInARow() throws Exception { + AtomicReference capturedSink = new AtomicReference<>(); + + DataSourceSynchronizerAdapter adapter = new DataSourceSynchronizerAdapter(sink -> { + capturedSink.set(sink); + return new MockDataSource(null, null); + }); + resourcesToClose.add(adapter); + + // Report a successful poll, a failed poll, and two more successful polls, as the FDv1 polling data source does. + CompletableFuture firstFuture = adapter.next(); + DataSourceUpdateSink sink = capturedSink.get(); + sink.init(new FullDataSet<>(DataBuilder.forStandardTypes().build().getData(), true, null)); + sink.updateStatus(DataSourceStatusProvider.State.VALID, null); + sink.updateStatus(DataSourceStatusProvider.State.INTERRUPTED, + DataSourceStatusProvider.ErrorInfo.fromHttpError(503)); + sink.init(new FullDataSet<>(DataBuilder.forStandardTypes().build().getData(), true, null)); + sink.updateStatus(DataSourceStatusProvider.State.VALID, null); + sink.init(new FullDataSet<>(DataBuilder.forStandardTypes().build().getData(), true, null)); + sink.updateStatus(DataSourceStatusProvider.State.VALID, null); + List results = new ArrayList<>(); + results.add(firstFuture.get(2, TimeUnit.SECONDS)); + for (int i = 0; i < 4; i++) { + results.add(adapter.next().get(2, TimeUnit.SECONDS)); + } + + // Only the last poll meets the reset threshold, and it reports the reset after its data. + assertEquals(SourceResultType.CHANGE_SET, results.get(0).getResultType()); + assertFalse(results.get(0).isResetThresholdMet()); + assertEquals(SourceSignal.INTERRUPTED, results.get(1).getStatus().getState()); + assertFalse(results.get(1).isResetThresholdMet()); + assertEquals(SourceResultType.CHANGE_SET, results.get(2).getResultType()); + assertFalse(results.get(2).isResetThresholdMet()); + assertEquals(SourceResultType.CHANGE_SET, results.get(3).getResultType()); + assertFalse(results.get(3).isResetThresholdMet()); + assertEquals(ChangeSetType.None, results.get(4).getChangeSet().getType()); + assertTrue(results.get(4).isResetThresholdMet()); + + adapter.close(); + } } diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceConditionsAggregateTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceConditionsAggregateTest.java deleted file mode 100644 index cae52f8b..00000000 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceConditionsAggregateTest.java +++ /dev/null @@ -1,238 +0,0 @@ -package com.launchdarkly.sdk.server; - -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.Condition; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.Condition.ConditionType; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.FallbackCondition; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.RecoveryCondition; -import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; -import com.launchdarkly.sdk.server.interfaces.DataSourceStatusProvider; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.time.Instant; -import java.util.Collections; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.sameInstance; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -/** - * Direct tests for {@link FDv2DataSource.Conditions}. - * - *

The Conditions class is the aggregator that races fallback/recovery - * condition futures against synchronizer.next() in the FDv2DataSource run - * loop. Each iteration of that loop calls getFuture() and passes the result to - * CompletableFuture.anyOf(...) -- so getFuture() must not return a shared - * instance, or every anyOf call permanently attaches a Completion node to the - * shared instance's stack, leaking memory proportional to event rate during - * the synchronizer's tenure on a healthy primary. - */ -public class FDv2DataSourceConditionsAggregateTest { - private ScheduledExecutorService executor; - - @Before - public void setUp() { - executor = Executors.newScheduledThreadPool(1); - } - - @After - public void tearDown() { - executor.shutdownNow(); - } - - /** - * Bug-proving test: getFuture() must return a fresh instance per call. - * - *

If it returns the same instance (as it did before the fix), the run - * loop's per-iteration {@code anyOf(getFuture(), syncNext)} attaches a new - * OrRelay Completion to the shared future's stack every iteration, with no - * deregister path -- a monotonic leak for a non-firing aggregate. - */ - @Test - public void getFutureReturnsDistinctInstancesPerCall() { - Condition fallback = new FallbackCondition(executor, 60); - try (FDv2DataSource.Conditions conditions = - new FDv2DataSource.Conditions(Collections.singletonList(fallback))) { - CompletableFuture f1 = conditions.getFuture(); - CompletableFuture f2 = conditions.getFuture(); - CompletableFuture f3 = conditions.getFuture(); - assertThat(f1, not(sameInstance(f2))); - assertThat(f2, not(sameInstance(f3))); - assertThat(f1, not(sameInstance(f3))); - } - } - - /** - * Even with no underlying conditions (a single-synchronizer configuration), - * getFuture() must return fresh instances. The aggregate never completes - * in this case, which is exactly the scenario where any per-iteration - * accumulation would be most damaging. - */ - @Test - public void getFutureReturnsDistinctInstancesEvenWithNoConditions() { - try (FDv2DataSource.Conditions conditions = - new FDv2DataSource.Conditions(Collections.emptyList())) { - CompletableFuture f1 = conditions.getFuture(); - CompletableFuture f2 = conditions.getFuture(); - assertThat(f1, not(sameInstance(f2))); - } - } - - /** - * Every fresh future returned by getFuture() must complete when the - * underlying aggregate fires. The fan-out via the single permanent listener - * is what makes the fresh-per-call pattern work; verify it actually - * delivers. - */ - @Test - public void allFreshFuturesCompleteWhenAggregateFires() throws Exception { - // 0-second timeout -> fires on first INTERRUPTED inform. - Condition fallback = new FallbackCondition(executor, 0); - try (FDv2DataSource.Conditions conditions = - new FDv2DataSource.Conditions(Collections.singletonList(fallback))) { - CompletableFuture f1 = conditions.getFuture(); - CompletableFuture f2 = conditions.getFuture(); - CompletableFuture f3 = conditions.getFuture(); - - conditions.inform(makeInterruptedResult()); - - Object r1 = f1.get(2, TimeUnit.SECONDS); - Object r2 = f2.get(2, TimeUnit.SECONDS); - Object r3 = f3.get(2, TimeUnit.SECONDS); - - assertNotNull(r1); - assertNotNull(r2); - assertNotNull(r3); - assertTrue(r1 instanceof Condition); - assertTrue(r2 instanceof Condition); - assertTrue(r3 instanceof Condition); - } - } - - /** - * Bug-proving test for the null-sentinel issue caught by Cursor Bugbot: - * if the underlying aggregate completes exceptionally, every - * future returned by getFuture() -- both those handed out before the - * exception and those requested after -- must complete exceptionally too. - * - *

Prior to this fix, the "fired" state was tracked via a {@code null} - * sentinel on a {@code completedValue} field, which also stayed - * {@code null} on exceptional completion. A subsequent getFuture() call - * would then return {@code CompletableFuture.completedFuture(null)} -- - * silently converting an exceptional completion into a normal - * {@code null} completion. The run loop's downstream - * {@code res.getClass().getName()} would then throw NPE. - */ - @Test - public void getFutureFailsExceptionallyWhenAggregateFailsExceptionally() - throws Exception { - ManualCondition manualCondition = new ManualCondition(); - try (FDv2DataSource.Conditions conditions = - new FDv2DataSource.Conditions(Collections.singletonList(manualCondition))) { - // Future requested BEFORE the exceptional completion. - CompletableFuture before = conditions.getFuture(); - - RuntimeException boom = new RuntimeException("simulated condition failure"); - manualCondition.future.completeExceptionally(boom); - - // Future requested AFTER the exceptional completion (exercises the - // fast path through makeCompletedFuture). This is the case bugbot - // caught: pre-fix, it returned completedFuture(null). - CompletableFuture after = conditions.getFuture(); - - assertThrowsExecutionExceptionWithCause(before, boom); - assertThrowsExecutionExceptionWithCause(after, boom); - } - } - - /** - * getFuture() called after the aggregate has already fired returns an - * already-completed future synchronously (the fast path). - */ - @Test - public void getFutureAfterAggregateFiresReturnsCompletedFuture() throws Exception { - // RecoveryCondition arms its timer in the constructor and fires after - // the configured timeout. With timeout=0 it fires near-immediately. - Condition recovery = new RecoveryCondition(executor, 0); - try (FDv2DataSource.Conditions conditions = - new FDv2DataSource.Conditions(Collections.singletonList(recovery))) { - // Drain a future to confirm the aggregate has fired. - conditions.getFuture().get(2, TimeUnit.SECONDS); - - CompletableFuture postFire = conditions.getFuture(); - assertTrue("post-fire getFuture() should be already complete", postFire.isDone()); - assertNotNull(postFire.get(0, TimeUnit.SECONDS)); - } - } - - private static FDv2SourceResult makeInterruptedResult() { - return FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo( - DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, - 0, - "simulated", - Instant.now()), - false); - } - - /** - * Asserts that {@code future.get()} throws {@link ExecutionException} - * wrapping the expected cause. {@code CompletableFuture#get} surfaces - * exceptional completion as ExecutionException with the original - * exception as its cause. - */ - private static void assertThrowsExecutionExceptionWithCause( - CompletableFuture future, - Throwable expectedCause) throws Exception { - try { - future.get(2, TimeUnit.SECONDS); - throw new AssertionError("expected ExecutionException, got normal completion"); - } catch (ExecutionException ee) { - if (ee.getCause() != expectedCause) { - throw new AssertionError( - "expected cause to be " + expectedCause + " but was " + ee.getCause(), ee); - } - } - } - - /** - * Test-only Condition with an externally-controllable future. The - * existing FallbackCondition/RecoveryCondition only resolve normally - * (with {@code this}); to exercise the exceptional path through the - * aggregate's whenComplete listener we need a Condition we can fail - * directly. - */ - private static final class ManualCondition - implements Condition { - final CompletableFuture future = new CompletableFuture<>(); - - @Override - public CompletableFuture execute() { - return future; - } - - @Override - public void inform(FDv2SourceResult sourceResult) { - // Manually controlled; no auto-trigger from inform. - } - - @Override - public void close() { - // No timer to cancel. - } - - @Override - public ConditionType getType() { - return ConditionType.FALLBACK; - } - } -} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java deleted file mode 100644 index 94bcf3bf..00000000 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceFallbackConditionTest.java +++ /dev/null @@ -1,446 +0,0 @@ -package com.launchdarkly.sdk.server; - -import com.launchdarkly.sdk.fdv2.Selector; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.Condition; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.FallbackCondition; -import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; -import com.launchdarkly.sdk.server.interfaces.DataSourceStatusProvider; -import com.launchdarkly.sdk.fdv2.ChangeSet; -import com.launchdarkly.sdk.fdv2.ChangeSetType; -import com.launchdarkly.sdk.server.subsystems.DataStoreTypes; -import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; -import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; -import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems; - -import org.junit.After; -import java.util.Map; - -import org.junit.Test; - -import java.time.Instant; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; - -public class FDv2DataSourceFallbackConditionTest extends BaseTest { - - private ScheduledExecutorService executor; - - @After - public void tearDown() { - if (executor != null && !executor.isShutdown()) { - executor.shutdownNow(); - } - } - - private ChangeSet>>> makeChangeSet() { - return new ChangeSet<>( - ChangeSetType.None, - Selector.EMPTY, - null, - null, - true - ); - } - - @Test - public void executeReturnsCompletableFuture() { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 120); - - CompletableFuture result = condition.execute(); - - assertFalse(result.isDone()); - } - - @Test - public void getTypeReturnsFallback() { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 120); - - assertEquals(Condition.ConditionType.FALLBACK, condition.getType()); - } - - @Test - public void interruptedStatusStartsTimerThatCompletesResultFuture() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); - - CompletableFuture resultFuture = condition.execute(); - assertFalse(resultFuture.isDone()); - - // Inform with INTERRUPTED status - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // Future should still not be done immediately - assertFalse(resultFuture.isDone()); - - // Wait for timeout to fire - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - - // Now it should be done and return the condition instance - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void changeSetCancelsActiveTimer() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); - - CompletableFuture resultFuture = condition.execute(); - - // Start timer with INTERRUPTED - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // Cancel timer with CHANGE_SET - ChangeSet>>> changeSet = makeChangeSet(); - condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - - // Wait longer than the timeout period - Thread.sleep(1500); - - // Future should still not be complete (timer was cancelled) - assertFalse(resultFuture.isDone()); - } - - @Test - public void changeSetWithoutActiveTimerDoesNothing() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with CHANGE_SET without starting a timer first - ChangeSet>>> changeSet = makeChangeSet(); - condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - - // Wait to ensure nothing happens - Thread.sleep(100); - - // Future should still not be complete - assertFalse(resultFuture.isDone()); - } - - @Test - public void multipleInterruptedStatusesDoNotStartMultipleTimers() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 2); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with INTERRUPTED multiple times - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - Thread.sleep(100); - - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - Thread.sleep(100); - - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // Wait for the timer (should only fire once) - Condition result = resultFuture.get(3, TimeUnit.SECONDS); - - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void terminalErrorStatusDoesNotStartTimer() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with TERMINAL_ERROR status - condition.inform( - FDv2SourceResult.terminalError( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.ERROR_RESPONSE, 401, null, Instant.now()), - false - ) - ); - - // Wait longer than timeout - Thread.sleep(1500); - - // Future should still not be complete (no timer started) - assertFalse(resultFuture.isDone()); - } - - @Test - public void shutdownStatusDoesNotStartTimer() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with SHUTDOWN status - condition.inform(FDv2SourceResult.shutdown()); - - // Wait longer than timeout - Thread.sleep(1500); - - // Future should still not be complete (no timer started) - assertFalse(resultFuture.isDone()); - } - - @Test - public void goodbyeStatusDoesNotStartTimer() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with GOODBYE status - condition.inform(FDv2SourceResult.goodbye("server-requested", false)); - - // Wait longer than timeout - Thread.sleep(1500); - - // Future should still not be complete (no timer started) - assertFalse(resultFuture.isDone()); - } - - @Test - public void closeCancelsActiveTimer() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); - - CompletableFuture resultFuture = condition.execute(); - - // Start timer with INTERRUPTED - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // Close the condition - condition.close(); - - // Wait longer than the timeout period - Thread.sleep(1500); - - // Future should still not be complete (timer was cancelled) - assertFalse(resultFuture.isDone()); - } - - @Test - public void closeWithoutActiveTimerDoesNotFail() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 120); - - // Close without starting a timer - condition.close(); - - // Should not throw exception - } - - @Test - public void timerCanBeStartedAfterBeingCancelled() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); - - CompletableFuture resultFuture = condition.execute(); - - // Start timer - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // Cancel timer with CHANGE_SET - ChangeSet>>> changeSet = makeChangeSet(); - condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - - // Start timer again - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // Wait for second timer to fire - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void changeSetAfterTimerFiresDoesNotAffectCompletedFuture() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); - - CompletableFuture resultFuture = condition.execute(); - - // Start timer - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // Wait for timer to fire - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - - // Inform with CHANGE_SET after timer has fired - ChangeSet>>> changeSet = makeChangeSet(); - condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - - // Future should remain complete - assertTrue(resultFuture.isDone()); - } - - @Test - public void factoryCreatesFallbackCondition() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition.Factory factory = new FallbackCondition.Factory(executor, 1); - - FallbackCondition condition = (FallbackCondition) factory.build(); - - // Verify it works by using it - CompletableFuture resultFuture = condition.execute(); - assertFalse(resultFuture.isDone()); - - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void executeReturnsTheSameFutureOnMultipleCalls() { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 120); - - CompletableFuture first = condition.execute(); - CompletableFuture second = condition.execute(); - - assertSame(first, second); - } - - @Test - public void changeSetDuringTimerExecutionCancelsTimer() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); - - CompletableFuture resultFuture = condition.execute(); - - // Start timer - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // Wait partway through timeout period - Thread.sleep(500); - - // Cancel with CHANGE_SET - ChangeSet>>> changeSet = makeChangeSet(); - condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - - // Wait past the original timeout - Thread.sleep(1000); - - // Future should still not be complete - assertFalse(resultFuture.isDone()); - } - - @Test - public void multipleChangeSetCallsWithActiveTimerAreHandled() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); - - CompletableFuture resultFuture = condition.execute(); - - // Start timer - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // Cancel with multiple CHANGE_SETs - ChangeSet>>> changeSet = makeChangeSet(); - condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - - // Wait longer than timeout - Thread.sleep(1500); - - // Future should still not be complete - assertFalse(resultFuture.isDone()); - } - - @Test - public void closeCanBeCalledMultipleTimes() throws Exception { - executor = Executors.newScheduledThreadPool(1); - FallbackCondition condition = new FallbackCondition(executor, 1); - - // Start timer - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // Close multiple times - condition.close(); - condition.close(); - condition.close(); - - // Should not throw exception - } -} \ No newline at end of file diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceRecoveryConditionTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceRecoveryConditionTest.java deleted file mode 100644 index b3878bad..00000000 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceRecoveryConditionTest.java +++ /dev/null @@ -1,329 +0,0 @@ -package com.launchdarkly.sdk.server; - -import com.launchdarkly.sdk.fdv2.Selector; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.Condition; -import com.launchdarkly.sdk.server.FDv2DataSourceConditions.RecoveryCondition; -import com.launchdarkly.sdk.server.datasources.FDv2SourceResult; -import com.launchdarkly.sdk.server.interfaces.DataSourceStatusProvider; -import com.launchdarkly.sdk.fdv2.ChangeSet; -import com.launchdarkly.sdk.fdv2.ChangeSetType; -import com.launchdarkly.sdk.server.subsystems.DataStoreTypes; -import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; -import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; -import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems; - -import org.junit.After; -import org.junit.Test; - -import java.time.Instant; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; - -@SuppressWarnings("javadoc") -public class FDv2DataSourceRecoveryConditionTest extends BaseTest { - - private ScheduledExecutorService executor; - - @After - public void tearDown() { - if (executor != null && !executor.isShutdown()) { - executor.shutdownNow(); - } - } - - private ChangeSet>>> makeChangeSet() { - return new ChangeSet<>( - ChangeSetType.None, - Selector.EMPTY, - null, - null, - true - ); - } - - @Test - public void getTypeReturnsRecovery() { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 120); - - assertEquals(Condition.ConditionType.RECOVERY, condition.getType()); - } - - @Test - public void timerStartsImmediatelyAndCompletesResultFuture() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); - - CompletableFuture resultFuture = condition.execute(); - - // Future should not be done immediately - assertFalse(resultFuture.isDone()); - - // Wait for timeout to fire - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - - // Now it should be done and return the condition instance - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void closeCancelsActiveTimer() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); - - CompletableFuture resultFuture = condition.execute(); - - // Close the condition before timeout - condition.close(); - - // Wait longer than the timeout period - Thread.sleep(1500); - - // Future should still not be complete (timer was cancelled) - assertFalse(resultFuture.isDone()); - } - - @Test - public void closeAfterTimerFiresDoesNotCauseError() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); - - CompletableFuture resultFuture = condition.execute(); - - // Wait for timer to fire - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - - // Close after timer has fired - condition.close(); - - // Should not throw exception - } - - @Test - public void closeCanBeCalledMultipleTimes() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); - - // Close multiple times before timer fires - condition.close(); - condition.close(); - condition.close(); - - // Should not throw exception - } - - @Test - public void informWithChangeSetDoesNothing() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with CHANGE_SET - ChangeSet>>> changeSet = makeChangeSet(); - condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - - // Timer should still fire after timeout (inform does nothing) - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void informWithInterruptedStatusDoesNothing() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with INTERRUPTED status - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - - // Timer should still fire after timeout (inform does nothing) - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void informWithTerminalErrorStatusDoesNothing() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with TERMINAL_ERROR status - condition.inform( - FDv2SourceResult.terminalError( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.ERROR_RESPONSE, 401, null, Instant.now()), - false - ) - ); - - // Timer should still fire after timeout (inform does nothing) - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void informWithShutdownStatusDoesNothing() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with SHUTDOWN status - condition.inform(FDv2SourceResult.shutdown()); - - // Timer should still fire after timeout (inform does nothing) - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void informWithGoodbyeStatusDoesNothing() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with GOODBYE status - condition.inform(FDv2SourceResult.goodbye("server-requested", false)); - - // Timer should still fire after timeout (inform does nothing) - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void informWithNullDoesNothing() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); - - CompletableFuture resultFuture = condition.execute(); - - // Inform with null - condition.inform(null); - - // Timer should still fire after timeout (inform does nothing) - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void multipleInformCallsDoNotAffectTimer() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); - - CompletableFuture resultFuture = condition.execute(); - - // Multiple inform calls - ChangeSet>>> changeSet = makeChangeSet(); - condition.inform(FDv2SourceResult.changeSet(changeSet, false)); - condition.inform( - FDv2SourceResult.interrupted( - new DataSourceStatusProvider.ErrorInfo(DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), - false - ) - ); - condition.inform(FDv2SourceResult.shutdown()); - - // Timer should still fire after timeout (all inform calls do nothing) - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void factoryCreatesRecoveryCondition() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition.Factory factory = new RecoveryCondition.Factory(executor, 1); - - RecoveryCondition condition = (RecoveryCondition) factory.build(); - - // Verify it works by using it - CompletableFuture resultFuture = condition.execute(); - assertFalse(resultFuture.isDone()); - - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - assertTrue(resultFuture.isDone()); - assertSame(condition, result); - } - - @Test - public void factoryGetTypeReturnsRecovery() { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition.Factory factory = new RecoveryCondition.Factory(executor, 1); - - assertEquals(Condition.ConditionType.RECOVERY, factory.getType()); - } - - @Test - public void executeReturnsTheSameFutureOnMultipleCalls() { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 120); - - CompletableFuture first = condition.execute(); - CompletableFuture second = condition.execute(); - - assertSame(first, second); - } - - @Test - public void timerStartsImmediatelyOnConstruction() throws Exception { - executor = Executors.newScheduledThreadPool(1); - - // Create condition with very short timeout - RecoveryCondition condition = new RecoveryCondition(executor, 1); - - // Get the future - CompletableFuture resultFuture = condition.execute(); - - // Verify it's not done yet - assertFalse(resultFuture.isDone()); - - // Wait for it to complete - Condition result = resultFuture.get(2, TimeUnit.SECONDS); - - // Verify it completed with the condition - assertSame(condition, result); - } - - @Test - public void closeBeforeExecuteDoesNotPreventFutureAccess() throws Exception { - executor = Executors.newScheduledThreadPool(1); - RecoveryCondition condition = new RecoveryCondition(executor, 1); - - // Close immediately - condition.close(); - - // Should still be able to get the future - CompletableFuture resultFuture = condition.execute(); - - // Wait to ensure timer doesn't fire - Thread.sleep(1500); - - // Future should not be complete (timer was cancelled) - assertFalse(resultFuture.isDone()); - } -} diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java index d2015819..81063522 100644 --- a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/FDv2DataSourceTest.java @@ -25,8 +25,10 @@ import org.junit.After; import org.junit.Test; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; @@ -38,15 +40,13 @@ @SuppressWarnings("javadoc") public class FDv2DataSourceTest extends BaseTest { - private ScheduledExecutorService executor; + private final ScheduledExecutorService executor = Executors.newScheduledThreadPool(2); private final LDLogger logger = LDLogger.withAdapter(Logs.none(), ""); private final List resourcesToClose = new ArrayList<>(); @After public void tearDown() { - if (executor != null && !executor.isShutdown()) { - executor.shutdownNow(); - } + executor.shutdownNow(); for (AutoCloseable resource : resourcesToClose) { try { resource.close(); @@ -98,7 +98,6 @@ private FDv2SourceResult makeTerminalErrorResult() { @Test public void firstInitializerFailsSecondInitializerSucceedsWithSelector() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture firstInitializerFuture = new CompletableFuture<>(); @@ -115,7 +114,13 @@ public void firstInitializerFailsSecondInitializerSucceedsWithSelector() throws ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -136,7 +141,6 @@ public void firstInitializerFailsSecondInitializerSucceedsWithSelector() throws @Test public void firstInitializerFailsSecondInitializerSucceedsWithoutSelector() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture firstInitializerFuture = new CompletableFuture<>(); @@ -163,7 +167,13 @@ public void firstInitializerFailsSecondInitializerSucceedsWithoutSelector() thro } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -191,7 +201,6 @@ public void firstInitializerFailsSecondInitializerSucceedsWithoutSelector() thro @Test public void firstInitializerSucceedsWithSelectorSecondInitializerNotInvoked() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture firstInitializerFuture = CompletableFuture.completedFuture( @@ -212,7 +221,13 @@ public void firstInitializerSucceedsWithSelectorSecondInitializerNotInvoked() th ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -231,7 +246,6 @@ public void firstInitializerSucceedsWithSelectorSecondInitializerNotInvoked() th @Test public void allInitializersFailSwitchesToSynchronizers() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture firstInitializerFuture = new CompletableFuture<>(); @@ -257,7 +271,13 @@ public void allInitializersFailSwitchesToSynchronizers() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -276,7 +296,6 @@ public void allInitializersFailSwitchesToSynchronizers() throws Exception { @Test public void allThreeInitializersFailWithNoSynchronizers() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of( @@ -299,7 +318,13 @@ public void allThreeInitializersFailWithNoSynchronizers() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -326,7 +351,6 @@ public void allThreeInitializersFailWithNoSynchronizers() throws Exception { @Test public void oneInitializerNoSynchronizerIsWellBehaved() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initializerFuture = CompletableFuture.completedFuture( @@ -339,7 +363,13 @@ public void oneInitializerNoSynchronizerIsWellBehaved() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -361,7 +391,6 @@ public void oneInitializerNoSynchronizerIsWellBehaved() throws Exception { @Test public void noInitializersOneSynchronizerIsWellBehaved() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -374,7 +403,13 @@ public void noInitializersOneSynchronizerIsWellBehaved() throws Exception { () -> new MockSynchronizer(synchronizerFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -389,7 +424,6 @@ public void noInitializersOneSynchronizerIsWellBehaved() throws Exception { @Test public void oneInitializerOneSynchronizerIsWellBehaved() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initializerFuture = CompletableFuture.completedFuture( @@ -408,7 +442,13 @@ public void oneInitializerOneSynchronizerIsWellBehaved() throws Exception { () -> new MockSynchronizer(synchronizerFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -423,13 +463,18 @@ public void oneInitializerOneSynchronizerIsWellBehaved() throws Exception { @Test public void noInitializersAndNoSynchronizersIsWellBehaved() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -451,7 +496,6 @@ public void noInitializersAndNoSynchronizersIsWellBehaved() throws Exception { @Test public void errorWithFDv1FallbackTriggersFallback() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -468,7 +512,13 @@ public void errorWithFDv1FallbackTriggersFallback() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -481,7 +531,6 @@ public void errorWithFDv1FallbackTriggersFallback() throws Exception { @Test public void fallbackAndRecoveryTasksWellBehaved() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -512,7 +561,13 @@ public void fallbackAndRecoveryTasksWellBehaved() throws Exception { ); // Use short timeouts for testing - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(1), + /* recoveryTimeout= */ Duration.ofSeconds(2), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -551,7 +606,6 @@ public void fallbackAndRecoveryTasksWellBehaved() throws Exception { @Test public void canDisposeWhenSynchronizersFallingBack() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -565,7 +619,13 @@ public void canDisposeWhenSynchronizersFallingBack() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(1), + /* recoveryTimeout= */ Duration.ofSeconds(2), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -580,12 +640,11 @@ public void canDisposeWhenSynchronizersFallingBack() throws Exception { } // ============================================================================ - // Source Blocking + // Source Parking // ============================================================================ @Test - public void terminalErrorBlocksSynchronizer() throws Exception { - executor = Executors.newScheduledThreadPool(2); + public void terminalErrorFailsOverToNextSynchronizer() throws Exception { MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -612,7 +671,13 @@ public void terminalErrorBlocksSynchronizer() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -625,13 +690,13 @@ public void terminalErrorBlocksSynchronizer() throws Exception { Integer secondCall = synchronizerCallQueue.poll(2, TimeUnit.SECONDS); assertNotNull("First synchronizer should be called", firstCall); - assertNotNull("Second synchronizer should be called after first is blocked", secondCall); + assertNotNull("Second synchronizer should be called after first is parked", secondCall); assertEquals(Integer.valueOf(1), firstCall); assertEquals(Integer.valueOf(2), secondCall); // Expected status sequence: // 1. VALID when first synchronizer sends initial changeset - // 2. Terminal error from first synchronizer (blocks it) + // 2. Terminal error from first synchronizer (parks it) // 3. VALID when second synchronizer sends changeset List statuses = sink.awaitStatuses(3, 2, TimeUnit.SECONDS); assertEquals("Should receive 3 status updates", 3, statuses.size()); @@ -645,47 +710,584 @@ public void terminalErrorBlocksSynchronizer() throws Exception { } @Test - public void allThreeSynchronizersFailReportsExhaustion() throws Exception { - executor = Executors.newScheduledThreadPool(2); + public void terminalErrorParksSynchronizerAndRestartsItAfterUnparkTime() throws Exception { MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); - // All synchronizers send terminal errors + // The first instance fails with a terminal error, and the second instance delivers data. + BlockingQueue firstInstanceResults = new LinkedBlockingQueue<>(); + firstInstanceResults.add(makeTerminalErrorResult()); + BlockingQueue secondInstanceResults = new LinkedBlockingQueue<>(); + secondInstanceResults.add(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + AtomicInteger buildCount = new AtomicInteger(); + + ImmutableList> synchronizers = ImmutableList.of( + () -> buildCount.incrementAndGet() == 1 + ? new MockQueuedSynchronizer(firstInstanceResults) + : new MockQueuedSynchronizer(secondInstanceResults) + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); + resourcesToClose.add(dataSource); + + // Start the data system with its only synchronizer. + dataSource.start().get(5, TimeUnit.SECONDS); + + // The parked synchronizer is built again after its unpark time, and its data initializes the data system. + assertEquals(2, buildCount.get()); + assertTrue(dataSource.isInitialized()); + assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); + } + + @Test + public void failedNextParksSynchronizer() throws Exception { + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger buildCount = new AtomicInteger(); + + // The first instance fails from next(), and the second instance delivers data. + ImmutableList> synchronizers = ImmutableList.of( + () -> buildCount.incrementAndGet() == 1 + ? new MockSynchronizer(() -> { + throw new RuntimeException("next-boom"); + }, "throwing-sync") + : new MockSynchronizer( + CompletableFuture.completedFuture(FDv2SourceResult.changeSet(makeChangeSet(true), false)), + "throwing-sync") + ); + + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, testLogger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); + resourcesToClose.add(dataSource); + + // Start the data system with its only synchronizer. + dataSource.start().get(5, TimeUnit.SECONDS); + + // The failing synchronizer is parked, and is built again after its unpark time. + assertEquals(2, buildCount.get()); + assertTrue(logTextContains(logCapture, LDLogLevel.WARN, "Synchronizer 'throwing-sync' is backing off for ")); + } + + @Test + public void exceptionThrownFromNextParksSynchronizer() throws Exception { + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger buildCount = new AtomicInteger(); + + ImmutableList> synchronizers = ImmutableList.of( + () -> { + if (buildCount.incrementAndGet() == 1) { + // The first instance throws from next() instead of returning a future. + return new Synchronizer() { + @Override + public CompletableFuture next() { + throw new RuntimeException("next-boom"); + } + + @Override + public void close() { + } + }; + } + // The second instance delivers data. + return new MockSynchronizer( + CompletableFuture.completedFuture(FDv2SourceResult.changeSet(makeChangeSet(true), false))); + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); + resourcesToClose.add(dataSource); + + // Start the data system with its only synchronizer. + dataSource.start().get(5, TimeUnit.SECONDS); + + // The throwing synchronizer is parked, and is built again after its unpark time. + assertEquals(2, buildCount.get()); + assertTrue(dataSource.isInitialized()); + } + + @Test + public void constructionFailureParksSynchronizer() throws Exception { + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger buildCount = new AtomicInteger(); + + ImmutableList> synchronizers = ImmutableList.of( + () -> { + if (buildCount.incrementAndGet() == 1) { + // The first build fails. + throw new RuntimeException("build-boom"); + } + // The second build delivers data. + return new MockSynchronizer( + CompletableFuture.completedFuture(FDv2SourceResult.changeSet(makeChangeSet(true), false))); + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); + resourcesToClose.add(dataSource); + + // Start the data system with its only synchronizer. + dataSource.start().get(5, TimeUnit.SECONDS); + + // The synchronizer that failed to build is parked, and is built again after its unpark time. + assertEquals(2, buildCount.get()); + assertTrue(dataSource.isInitialized()); + } + + @Test + public void resetThresholdMetReturnsParkDelayToBase() throws Exception { + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger buildCount = new AtomicInteger(); + BlockingQueue buildTimes = new LinkedBlockingQueue<>(); + + ImmutableList> synchronizers = ImmutableList.of( + () -> { + buildTimes.offer(System.nanoTime()); + BlockingQueue results = new LinkedBlockingQueue<>(); + if (buildCount.incrementAndGet() == 3) { + // The third instance reports that it met its reset threshold before it fails. + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), false).withResetThresholdMet()); + } + // Every instance fails with a terminal error. + results.add(makeTerminalErrorResult()); + return new MockQueuedSynchronizer(results); + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(10), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); + resourcesToClose.add(dataSource); + + // Start the data system, and record when the synchronizer is built for the third and fourth times. + dataSource.start(); + buildTimes.poll(5, TimeUnit.SECONDS); + buildTimes.poll(5, TimeUnit.SECONDS); + Long thirdBuild = buildTimes.poll(5, TimeUnit.SECONDS); + Long fourthBuild = buildTimes.poll(5, TimeUnit.SECONDS); + + // After the reset, the park is at most the base delay. Without the reset, it would be at least twice the base. + assertNotNull(fourthBuild); + long parkMillis = TimeUnit.NANOSECONDS.toMillis(fourthBuild - thirdBuild); + assertTrue("park lasted " + parkMillis + "ms", parkMillis < 200); + } + + @Test + public void fallbackWithNoOtherAvailableSynchronizerKeepsCurrentSynchronizer() throws Exception { + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger secondaryBuildCount = new AtomicInteger(); + IterableAsyncQueue secondaryResults = new IterableAsyncQueue<>(); + secondaryResults.put(makeInterruptedResult()); + ImmutableList> synchronizers = ImmutableList.of( + // The primary synchronizer fails, so it is parked for the rest of the test. () -> { BlockingQueue results = new LinkedBlockingQueue<>(); results.add(makeTerminalErrorResult()); return new MockQueuedSynchronizer(results); }, + // The first instance of the secondary synchronizer reports that it is interrupted. + () -> secondaryBuildCount.incrementAndGet() == 1 + ? new MockQueuedSynchronizer(secondaryResults) + : new MockQueuedSynchronizer(new IterableAsyncQueue<>()) + ); + + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofMillis(10), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); + resourcesToClose.add(dataSource); + + // Start the data system. After the fallback condition of the secondary synchronizer is met, send a change set + // to the instance of the secondary synchronizer that was already running. + dataSource.start(); + executor.schedule( + () -> secondaryResults.put(FDv2SourceResult.changeSet(makeChangeSet(true), false)), + 60, + TimeUnit.MILLISECONDS); + + // The data system applies the change set, so it kept that instance. + sink.awaitApplyCount(1, 2, TimeUnit.SECONDS); + assertEquals(1, sink.getApplyCount()); + assertEquals(1, secondaryBuildCount.get()); + } + + @Test + public void unparkWhileCurrentSynchronizerIsHealthyKeepsIt() throws Exception { + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger primaryBuildCount = new AtomicInteger(); + IterableAsyncQueue secondaryResults = new IterableAsyncQueue<>(); + secondaryResults.put(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + + ImmutableList> synchronizers = ImmutableList.of( + // The primary synchronizer fails, so it is parked briefly. () -> { + primaryBuildCount.incrementAndGet(); BlockingQueue results = new LinkedBlockingQueue<>(); results.add(makeTerminalErrorResult()); return new MockQueuedSynchronizer(results); }, + // The secondary synchronizer delivers data and stays healthy. + () -> new MockQueuedSynchronizer(secondaryResults) + ); + + // The primary synchronizer is parked for 25-50ms. The fallback and recovery timeouts are too long to matter. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(50), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); + resourcesToClose.add(dataSource); + + // Start the data system, and send another change set after the primary synchronizer unparks. + dataSource.start(); + executor.schedule( + () -> secondaryResults.put(FDv2SourceResult.changeSet(makeChangeSet(true), false)), + 100, + TimeUnit.MILLISECONDS); + + // The data system applies both change sets, and does not start the primary synchronizer again. + sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); + assertEquals(2, sink.getApplyCount()); + assertEquals(1, primaryBuildCount.get()); + } + + @Test + public void fallbackSwitchesWhenParkedSynchronizerUnparks() throws Exception { + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger primaryBuildCount = new AtomicInteger(); + AtomicInteger secondaryBuildCount = new AtomicInteger(); + + ImmutableList> synchronizers = ImmutableList.of( + // The first instance of the primary synchronizer fails, and the second instance delivers data. + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(primaryBuildCount.incrementAndGet() == 1 + ? makeTerminalErrorResult() + : FDv2SourceResult.changeSet(makeChangeSet(true), false)); + return new MockQueuedSynchronizer(results); + }, + // The secondary synchronizer reports that it is interrupted. + () -> { + secondaryBuildCount.incrementAndGet(); + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(makeInterruptedResult()); + return new MockQueuedSynchronizer(results); + } + ); + + // The fallback condition is met after 10ms, and the primary synchronizer unparks after 50-100ms. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofMillis(10), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); + resourcesToClose.add(dataSource); + + // Start the data system. + dataSource.start(); + + // When the primary synchronizer unparks, the data system falls back to it and applies its data. + sink.awaitApplyCount(1, 2, TimeUnit.SECONDS); + assertEquals(2, primaryBuildCount.get()); + assertEquals(1, secondaryBuildCount.get()); + } + + @Test + public void changeSetAfterFallbackIsMetPreventsSwitchAtUnpark() throws Exception { + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger primaryBuildCount = new AtomicInteger(); + IterableAsyncQueue secondaryResults = new IterableAsyncQueue<>(); + secondaryResults.put(makeInterruptedResult()); + + ImmutableList> synchronizers = ImmutableList.of( + // The primary synchronizer fails, so it is parked briefly. () -> { + primaryBuildCount.incrementAndGet(); BlockingQueue results = new LinkedBlockingQueue<>(); results.add(makeTerminalErrorResult()); return new MockQueuedSynchronizer(results); + }, + // The secondary synchronizer reports that it is interrupted. + () -> new MockQueuedSynchronizer(secondaryResults) + ); + + // The fallback condition is met after 10ms, and the primary synchronizer unparks after 80-160ms. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofMillis(10), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(160), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); + resourcesToClose.add(dataSource); + + // Start the data system, send a change set after the fallback is met, and another after the unpark. + dataSource.start(); + executor.schedule( + () -> secondaryResults.put(FDv2SourceResult.changeSet(makeChangeSet(true), false)), + 50, + TimeUnit.MILLISECONDS); + executor.schedule( + () -> secondaryResults.put(FDv2SourceResult.changeSet(makeChangeSet(true), false)), + 210, + TimeUnit.MILLISECONDS); + + // The data system applies both change sets, and does not start the primary synchronizer again. + sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); + assertEquals(2, sink.getApplyCount()); + assertEquals(1, primaryBuildCount.get()); + } + + @Test + public void recoveryWithPreferredSynchronizerParkedWaitsForItsUnpark() throws Exception { + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger primaryBuildCount = new AtomicInteger(); + AtomicInteger secondaryBuildCount = new AtomicInteger(); + + ImmutableList> synchronizers = ImmutableList.of( + // The first instance of the primary synchronizer fails, and the second instance delivers data. + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(primaryBuildCount.incrementAndGet() == 1 + ? makeTerminalErrorResult() + : FDv2SourceResult.changeSet(makeChangeSet(true), false)); + return new MockQueuedSynchronizer(results); + }, + // The secondary synchronizer delivers data and stays healthy. + () -> { + secondaryBuildCount.incrementAndGet(); + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + return new MockQueuedSynchronizer(results); } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + // The recovery condition is met after 50ms, and the primary synchronizer unparks after 100-200ms. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofMillis(50), + /* parkInitialDelay= */ Duration.ofMillis(200), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); - Future startFuture = dataSource.start(); - startFuture.get(2, TimeUnit.SECONDS); + // Start the data system. + dataSource.start(); - // Expected status sequence: 3 terminal errors (one per synchronizer) → OFF when all exhausted - // Terminal errors might show as INTERRUPTED status - List statuses = sink.awaitStatuses(4, 2, TimeUnit.SECONDS); - assertEquals("Should receive 4 status updates", 4, statuses.size()); - // Last status should be OFF - assertEquals(DataSourceStatusProvider.State.OFF, statuses.get(3)); + // When the primary synchronizer unparks, the data system recovers to it and applies its data. + sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); + assertEquals(2, primaryBuildCount.get()); + assertEquals(1, secondaryBuildCount.get()); + } - assertFalse(dataSource.isInitialized()); - assertEquals(DataSourceStatusProvider.State.OFF, sink.getLastState()); - assertNotNull(sink.getLastError()); + @Test + public void changeSetDuringInterruptionRestartsFallbackTimeout() throws Exception { + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + IterableAsyncQueue primaryResults = new IterableAsyncQueue<>(); + primaryResults.put(makeInterruptedResult()); + BlockingQueue secondaryBuildTimes = new LinkedBlockingQueue<>(); + + ImmutableList> synchronizers = ImmutableList.of( + // The primary synchronizer reports that it is interrupted. + () -> new MockQueuedSynchronizer(primaryResults), + // The secondary synchronizer records when it is built. + () -> { + secondaryBuildTimes.offer(System.nanoTime()); + return new MockQueuedSynchronizer(new IterableAsyncQueue<>()); + } + ); + + // The fallback timeout is 100ms. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofMillis(100), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); + resourcesToClose.add(dataSource); + + // Start the data system. Before the fallback timeout, the primary delivers a change set and is interrupted again. + long start = System.nanoTime(); + dataSource.start(); + executor.schedule( + () -> primaryResults.put(FDv2SourceResult.changeSet(makeChangeSet(true), false)), + 50, + TimeUnit.MILLISECONDS); + executor.schedule(() -> primaryResults.put(makeInterruptedResult()), 90, TimeUnit.MILLISECONDS); + + // The data system falls back a full timeout after the second interruption, not at the original deadline. + Long secondaryBuild = secondaryBuildTimes.poll(2, TimeUnit.SECONDS); + assertNotNull(secondaryBuild); + long fallbackMillis = TimeUnit.NANOSECONDS.toMillis(secondaryBuild - start); + assertTrue("fell back after " + fallbackMillis + "ms", fallbackMillis >= 150); + } + + // A synchronizer that shuts down without being closed is parked, like one with any other unexpected failure. + @Test + public void unpromptedShutdownParksSynchronizer() throws Exception { + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger buildCount = new AtomicInteger(); + + // The first instance shuts down, and the second instance delivers data. + ImmutableList> synchronizers = ImmutableList.of( + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(buildCount.incrementAndGet() == 1 + ? FDv2SourceResult.shutdown() + : FDv2SourceResult.changeSet(makeChangeSet(true), false)); + return new MockQueuedSynchronizer(results); + } + ); + + // The synchronizer is parked for 50-100ms. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); + resourcesToClose.add(dataSource); + + // Start the data system. + dataSource.start().get(2, TimeUnit.SECONDS); + + // The synchronizer is built again after its unpark time, and its data initializes the data system. + assertEquals(2, buildCount.get()); + assertTrue(dataSource.isInitialized()); + } + + // While every synchronizer is parked, the synchronizer that was parked last is stopped. + @Test + public void parkedSynchronizerIsClosedWhileAllSynchronizersAreParked() throws Exception { + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + CountDownLatch closed = new CountDownLatch(1); + + // The only synchronizer fails with a terminal error, and records when it is closed. + ImmutableList> synchronizers = ImmutableList.of( + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(makeTerminalErrorResult()); + return new MockQueuedSynchronizer(results) { + @Override + public void close() { + closed.countDown(); + super.close(); + } + }; + } + ); + + // The synchronizer is parked for much longer than the test. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); + resourcesToClose.add(dataSource); + + // Start the data system. + dataSource.start(); + + // The parked synchronizer is closed while it waits for its unpark time. + assertTrue(closed.await(1, TimeUnit.SECONDS)); + } + + @Test + public void allSynchronizersParkedRestartsAfterEarliestUnparkTime() throws Exception { + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + // Every instance of every synchronizer fails with a terminal error. + BlockingQueue buildOrder = new LinkedBlockingQueue<>(); + ImmutableList> synchronizers = ImmutableList.of( + () -> { + buildOrder.offer(1); + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(makeTerminalErrorResult()); + return new MockQueuedSynchronizer(results); + }, + () -> { + buildOrder.offer(2); + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(makeTerminalErrorResult()); + return new MockQueuedSynchronizer(results); + }, + () -> { + buildOrder.offer(3); + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(makeTerminalErrorResult()); + return new MockQueuedSynchronizer(results); + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); + resourcesToClose.add(dataSource); + + // Start the data system, and wait for each synchronizer to be built and then for one more build. + dataSource.start(); + Integer firstBuild = buildOrder.poll(2, TimeUnit.SECONDS); + Integer secondBuild = buildOrder.poll(2, TimeUnit.SECONDS); + Integer thirdBuild = buildOrder.poll(2, TimeUnit.SECONDS); + Integer buildAfterAllParked = buildOrder.poll(2, TimeUnit.SECONDS); + + // The synchronizers fail over in order, and a parked synchronizer starts again after all of them are parked. + assertEquals(Integer.valueOf(1), firstBuild); + assertEquals(Integer.valueOf(2), secondBuild); + assertEquals(Integer.valueOf(3), thirdBuild); + assertNotNull(buildAfterAllParked); + + // Each park reports INTERRUPTED. + List statuses = sink.awaitStatuses(3, 2, TimeUnit.SECONDS); + assertEquals(Arrays.asList( + DataSourceStatusProvider.State.INTERRUPTED, + DataSourceStatusProvider.State.INTERRUPTED, + DataSourceStatusProvider.State.INTERRUPTED), statuses); } // ============================================================================ @@ -694,7 +1296,6 @@ public void allThreeSynchronizersFailReportsExhaustion() throws Exception { @Test public void disabledDataSourceCannotTriggerActions() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -718,7 +1319,13 @@ public void disabledDataSourceCannotTriggerActions() throws Exception { () -> new MockQueuedSynchronizer(secondSyncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -751,7 +1358,6 @@ public void disabledDataSourceCannotTriggerActions() throws Exception { @Test public void disposeCompletesStartFuture() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -764,7 +1370,13 @@ public void disposeCompletesStartFuture() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -785,13 +1397,18 @@ public void disposeCompletesStartFuture() throws Exception { @Test public void noSourcesProvidedCompletesImmediately() throws Exception{ - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -812,7 +1429,6 @@ public void noSourcesProvidedCompletesImmediately() throws Exception{ @Test public void startFutureCompletesExactlyOnce() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initializerFuture = CompletableFuture.completedFuture( @@ -831,7 +1447,13 @@ public void startFutureCompletesExactlyOnce() throws Exception { () -> new MockSynchronizer(synchronizerFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -843,7 +1465,6 @@ public void startFutureCompletesExactlyOnce() throws Exception { @Test public void concurrentCloseAndStartHandledSafely() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); BlockingQueue syncResults = new LinkedBlockingQueue<>(); @@ -854,7 +1475,13 @@ public void concurrentCloseAndStartHandledSafely() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); Future startFuture = dataSource.start(); @@ -867,7 +1494,6 @@ public void concurrentCloseAndStartHandledSafely() throws Exception { @Test public void multipleStartCallsEventuallyComplete() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); BlockingQueue syncResults = new LinkedBlockingQueue<>(); @@ -878,7 +1504,13 @@ public void multipleStartCallsEventuallyComplete() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture1 = dataSource.start(); @@ -895,7 +1527,6 @@ public void multipleStartCallsEventuallyComplete() throws Exception { @Test public void isInitializedThreadSafe() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); BlockingQueue syncResults = new LinkedBlockingQueue<>(); @@ -906,7 +1537,13 @@ public void isInitializedThreadSafe() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); dataSource.start(); @@ -928,7 +1565,6 @@ public void isInitializedThreadSafe() throws Exception { @Test public void dataSourceUpdatesApplyThreadSafe() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); BlockingQueue syncResults = new LinkedBlockingQueue<>(); @@ -941,7 +1577,13 @@ public void dataSourceUpdatesApplyThreadSafe() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -960,7 +1602,6 @@ public void dataSourceUpdatesApplyThreadSafe() throws Exception { @Test public void initializerThrowsExecutionException() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture badFuture = new CompletableFuture<>(); @@ -977,7 +1618,13 @@ public void initializerThrowsExecutionException() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -989,7 +1636,6 @@ public void initializerThrowsExecutionException() throws Exception { @Test public void initializerThrowsInterruptedException() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); AtomicBoolean firstCalled = new AtomicBoolean(false); @@ -1009,7 +1655,13 @@ public void initializerThrowsInterruptedException() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1022,7 +1674,6 @@ public void initializerThrowsInterruptedException() throws Exception { @Test public void initializerThrowsCancellationException() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture cancelledFuture = new CompletableFuture<>(); @@ -1039,7 +1690,13 @@ public void initializerThrowsCancellationException() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1051,7 +1708,6 @@ public void initializerThrowsCancellationException() throws Exception { @Test public void synchronizerNextThrowsExecutionException() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -1068,7 +1724,13 @@ public void synchronizerNextThrowsExecutionException() throws Exception { () -> new MockSynchronizer(goodFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1080,7 +1742,6 @@ public void synchronizerNextThrowsExecutionException() throws Exception { @Test public void synchronizerNextThrowsInterruptedException() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -1100,7 +1761,13 @@ public void synchronizerNextThrowsInterruptedException() throws Exception { () -> new MockSynchronizer(goodFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1113,7 +1780,6 @@ public void synchronizerNextThrowsInterruptedException() throws Exception { @Test public void synchronizerNextThrowsCancellationException() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -1130,7 +1796,13 @@ public void synchronizerNextThrowsCancellationException() throws Exception { () -> new MockSynchronizer(goodFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1146,13 +1818,18 @@ public void synchronizerNextThrowsCancellationException() throws Exception { @Test public void closeWithoutStartDoesNotHang() { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); dataSource.close(); @@ -1161,7 +1838,6 @@ public void closeWithoutStartDoesNotHang() { @Test public void closeAfterInitializersCompletesImmediately() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initializerFuture = CompletableFuture.completedFuture( @@ -1174,7 +1850,13 @@ public void closeAfterInitializersCompletesImmediately() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1185,7 +1867,6 @@ public void closeAfterInitializersCompletesImmediately() throws Exception { @Test public void closeWhileSynchronizerRunningShutdownsSource() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -1205,7 +1886,13 @@ public void close() { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1217,7 +1904,6 @@ public void close() { @Test public void multipleCloseCallsAreIdempotent() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initializerFuture = CompletableFuture.completedFuture( @@ -1230,7 +1916,13 @@ public void multipleCloseCallsAreIdempotent() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1244,7 +1936,6 @@ public void multipleCloseCallsAreIdempotent() throws Exception { @Test public void closingDataSourceDuringInitializationReportsOffWithoutErrors() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initializerFuture = new CompletableFuture<>(); @@ -1256,7 +1947,13 @@ public void closingDataSourceDuringInitializationReportsOffWithoutErrors() throw ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1282,7 +1979,6 @@ public void closingDataSourceDuringInitializationReportsOffWithoutErrors() throw @Test public void dataSourceClosedDuringSynchronizationReportsOffWithoutError() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -1296,7 +1992,13 @@ public void dataSourceClosedDuringSynchronizationReportsOffWithoutError() throws } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1317,9 +2019,64 @@ public void dataSourceClosedDuringSynchronizationReportsOffWithoutError() throws assertTrue(dataSource.isInitialized()); // Was initialized before close } + // A status that the data source reports while it closes does not replace OFF. + @Test + public void closeWhileReportingStatusEndsWithOff() throws Exception { + CountDownLatch interruptedReporting = new CountDownLatch(1); + CountDownLatch finishReporting = new CountDownLatch(1); + CountDownLatch offReported = new CountDownLatch(1); + + // The sink holds the run thread inside the first INTERRUPTED report until the test releases it. + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink() { + @Override + public void updateStatus(DataSourceStatusProvider.State newState, DataSourceStatusProvider.ErrorInfo errorInfo) { + if (newState == DataSourceStatusProvider.State.INTERRUPTED && interruptedReporting.getCount() > 0) { + interruptedReporting.countDown(); + try { + finishReporting.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + super.updateStatus(newState, errorInfo); + if (newState == DataSourceStatusProvider.State.OFF) { + offReported.countDown(); + } + } + }; + + // The synchronizer reports an interruption. + ImmutableList> synchronizers = ImmutableList.of( + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(makeInterruptedResult()); + return new MockQueuedSynchronizer(results); + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); + resourcesToClose.add(dataSource); + + // Start the data source, close it while it reports INTERRUPTED, and then let it finish the report. + dataSource.start(); + assertTrue(interruptedReporting.await(2, TimeUnit.SECONDS)); + Future closeFuture = executor.submit(dataSource::close); + offReported.await(500, TimeUnit.MILLISECONDS); + finishReporting.countDown(); + closeFuture.get(5, TimeUnit.SECONDS); + + // The last status is OFF. + assertEquals(DataSourceStatusProvider.State.OFF, sink.getLastState()); + } + @Test public void closeInterruptsConditionWaiting() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -1333,7 +2090,13 @@ public void closeInterruptsConditionWaiting() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1350,7 +2113,6 @@ public void closeInterruptsConditionWaiting() throws Exception { @Test public void setActiveSourceReturnsShutdownStatus() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initializerFuture = CompletableFuture.completedFuture( @@ -1371,7 +2133,13 @@ public CompletableFuture run() { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); dataSource.close(); Future startFuture = dataSource.start(); @@ -1383,7 +2151,6 @@ public CompletableFuture run() { @Test public void activeSourceClosedWhenSwitchingSynchronizers() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -1408,7 +2175,13 @@ public void close() { () -> new MockQueuedSynchronizer(secondSyncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1420,21 +2193,99 @@ public void close() { assertTrue(firstSyncClosed.get()); } + // A synchronizer that shuts down by itself is parked with an error that describes the shutdown. @Test - public void activeSourceClosedOnShutdown() throws Exception { - executor = Executors.newScheduledThreadPool(2); + public void synchronizerShutdownByItselfReportsError() throws Exception { MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + CountDownLatch secondSyncStarted = new CountDownLatch(1); - ImmutableList> initializers = ImmutableList.of(); + ImmutableList> synchronizers = ImmutableList.of( + // The first synchronizer shuts down by itself. + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(FDv2SourceResult.shutdown()); + return new MockQueuedSynchronizer(results); + }, + // The second synchronizer records when it starts. + () -> { + secondSyncStarted.countDown(); + return new MockQueuedSynchronizer(new LinkedBlockingQueue<>()); + } + ); - BlockingQueue syncResults = new LinkedBlockingQueue<>(); - syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); + resourcesToClose.add(dataSource); - AtomicBoolean syncClosed = new AtomicBoolean(false); + // Start the data source, and wait until it moves on to the second synchronizer. + dataSource.start(); + assertTrue(secondSyncStarted.await(2, TimeUnit.SECONDS)); - ImmutableList> synchronizers = ImmutableList.of( - () -> new MockQueuedSynchronizer(syncResults) { - @Override + // The data source reported INTERRUPTED with an error. + assertEquals(DataSourceStatusProvider.State.INTERRUPTED, sink.getLastState()); + assertNotNull(sink.getLastError()); + } + + // A synchronizer that throws from close does not stop the data source from starting the next synchronizer. + @Test + public void exceptionFromSynchronizerCloseDoesNotStopSynchronization() throws Exception { + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + CountDownLatch secondSyncStarted = new CountDownLatch(1); + + ImmutableList> synchronizers = ImmutableList.of( + // The first synchronizer reports a terminal error, and throws when the data source closes it. + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(makeTerminalErrorResult()); + return new MockQueuedSynchronizer(results) { + @Override + public void close() { + throw new IllegalStateException("close failed"); + } + }; + }, + // The second synchronizer records when it starts. + () -> { + secondSyncStarted.countDown(); + return new MockQueuedSynchronizer(new LinkedBlockingQueue<>()); + } + ); + + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); + resourcesToClose.add(dataSource); + + // Start the data source. + dataSource.start(); + + // The data source starts the second synchronizer. + assertTrue(secondSyncStarted.await(2, TimeUnit.SECONDS)); + } + + @Test + public void activeSourceClosedOnShutdown() throws Exception { + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + + ImmutableList> initializers = ImmutableList.of(); + + BlockingQueue syncResults = new LinkedBlockingQueue<>(); + syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + + AtomicBoolean syncClosed = new AtomicBoolean(false); + + ImmutableList> synchronizers = ImmutableList.of( + () -> new MockQueuedSynchronizer(syncResults) { + @Override public void close() { syncClosed.set(true); super.close(); @@ -1442,7 +2293,13 @@ public void close() { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); @@ -1454,7 +2311,6 @@ public void close() { @Test public void setActiveSourceOnInitializerChecksShutdown() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CountDownLatch initializerStarted = new CountDownLatch(1); @@ -1474,7 +2330,13 @@ public void setActiveSourceOnInitializerChecksShutdown() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); Future startFuture = dataSource.start(); @@ -1498,13 +2360,12 @@ public void setActiveSourceOnInitializerChecksShutdown() throws Exception { // ============================================================================ @Test - public void blockedSynchronizerSkippedInRotation() throws Exception { - executor = Executors.newScheduledThreadPool(2); + public void parkedSynchronizerSkippedInRotation() throws Exception { MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); - // First: terminal error (blocked) + // First: terminal error (parked) BlockingQueue firstSyncResults = new LinkedBlockingQueue<>(); firstSyncResults.add(makeTerminalErrorResult()); @@ -1535,48 +2396,24 @@ public void blockedSynchronizerSkippedInRotation() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); - assertEquals(1, firstCallCount.get()); // Called once, then blocked + assertEquals(1, firstCallCount.get()); // Called once, then parked assertTrue(secondCallCount.get() >= 1); // Called } - @Test - public void allSynchronizersBlockedReturnsNullAndExits() throws Exception { - executor = Executors.newScheduledThreadPool(2); - MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); - - ImmutableList> initializers = ImmutableList.of(); - - ImmutableList> synchronizers = ImmutableList.of( - () -> { - BlockingQueue results = new LinkedBlockingQueue<>(); - results.add(makeTerminalErrorResult()); - return new MockQueuedSynchronizer(results); - }, - () -> { - BlockingQueue results = new LinkedBlockingQueue<>(); - results.add(makeTerminalErrorResult()); - return new MockQueuedSynchronizer(results); - } - ); - - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); - resourcesToClose.add(dataSource); - - Future startFuture = dataSource.start(); - startFuture.get(2, TimeUnit.SECONDS); - - assertFalse(dataSource.isInitialized()); - } - @Test public void recoveryResetsToFirstAvailableSynchronizer() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -1605,7 +2442,13 @@ public void recoveryResetsToFirstAvailableSynchronizer() throws Exception { ); // Short recovery timeout - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(1), + /* recoveryTimeout= */ Duration.ofSeconds(2), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1620,7 +2463,6 @@ public void recoveryResetsToFirstAvailableSynchronizer() throws Exception { @Test public void fallbackMovesToNextSynchronizer() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -1645,7 +2487,13 @@ public void fallbackMovesToNextSynchronizer() throws Exception { ); // Short fallback timeout - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(1), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1660,35 +2508,8 @@ public void fallbackMovesToNextSynchronizer() throws Exception { // Condition Lifecycle // ============================================================================ - @Test - public void conditionsClosedAfterSynchronizerLoop() throws Exception { - executor = Executors.newScheduledThreadPool(2); - MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); - - ImmutableList> initializers = ImmutableList.of(); - - BlockingQueue syncResults = new LinkedBlockingQueue<>(); - syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); - syncResults.add(makeTerminalErrorResult()); - - ImmutableList> synchronizers = ImmutableList.of( - () -> new MockQueuedSynchronizer(syncResults) - ); - - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); - resourcesToClose.add(dataSource); - - Future startFuture = dataSource.start(); - startFuture.get(2, TimeUnit.SECONDS); - - dataSource.close(); - - // If conditions weren't closed properly, we might see issues - } - @Test public void conditionsInformedOfAllResults() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -1702,7 +2523,13 @@ public void conditionsInformedOfAllResults() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 10, 20); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(10), + /* recoveryTimeout= */ Duration.ofSeconds(20), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1713,116 +2540,8 @@ public void conditionsInformedOfAllResults() throws Exception { assertTrue(sink.getApplyCount() >= 2); } - @Test - public void conditionsClosedOnException() throws Exception { - executor = Executors.newScheduledThreadPool(2); - MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); - - ImmutableList> initializers = ImmutableList.of(); - - CompletableFuture exceptionFuture = new CompletableFuture<>(); - exceptionFuture.completeExceptionally(new RuntimeException("Error")); - - ImmutableList> synchronizers = ImmutableList.of( - () -> new MockSynchronizer(exceptionFuture), - () -> new MockSynchronizer(CompletableFuture.completedFuture( - FDv2SourceResult.changeSet(makeChangeSet(false), false) - )) - ); - - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); - resourcesToClose.add(dataSource); - - Future startFuture = dataSource.start(); - startFuture.get(2, TimeUnit.SECONDS); - - // Conditions should be closed despite exception - } - - @Test - public void primeSynchronizerHasNoRecoveryCondition() throws Exception { - executor = Executors.newScheduledThreadPool(2); - MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); - - ImmutableList> initializers = ImmutableList.of(); - - BlockingQueue syncResults = new LinkedBlockingQueue<>(); - syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); - // Keep alive - - ImmutableList> synchronizers = ImmutableList.of( - () -> new MockQueuedSynchronizer(syncResults), - () -> new MockQueuedSynchronizer(new LinkedBlockingQueue<>()) - ); - - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); - resourcesToClose.add(dataSource); - - Future startFuture = dataSource.start(); - startFuture.get(2, TimeUnit.SECONDS); - - // Prime synchronizer should not have a recovery condition - // This is tested implicitly by the implementation - } - - @Test - public void nonPrimeSynchronizerHasBothConditions() throws Exception { - executor = Executors.newScheduledThreadPool(2); - MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); - - ImmutableList> initializers = ImmutableList.of(); - - // First: send INTERRUPTED to trigger fallback - BlockingQueue firstSyncResults = new LinkedBlockingQueue<>(); - firstSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); - firstSyncResults.add(makeInterruptedResult()); - - // Second: will have both conditions - BlockingQueue secondSyncResults = new LinkedBlockingQueue<>(); - secondSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); - - ImmutableList> synchronizers = ImmutableList.of( - () -> new MockQueuedSynchronizer(firstSyncResults), - () -> new MockQueuedSynchronizer(secondSyncResults) - ); - - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); - resourcesToClose.add(dataSource); - - Future startFuture = dataSource.start(); - startFuture.get(2, TimeUnit.SECONDS); - - // Non-prime synchronizer should have both fallback and recovery - // This is tested implicitly by the implementation - } - - @Test - public void singleSynchronizerHasNoConditions() throws Exception { - executor = Executors.newScheduledThreadPool(2); - MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); - - ImmutableList> initializers = ImmutableList.of(); - - BlockingQueue syncResults = new LinkedBlockingQueue<>(); - syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); - - ImmutableList> synchronizers = ImmutableList.of( - () -> new MockQueuedSynchronizer(syncResults) - ); - - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); - resourcesToClose.add(dataSource); - - Future startFuture = dataSource.start(); - startFuture.get(2, TimeUnit.SECONDS); - - // Single synchronizer should have no conditions - // This is tested implicitly by the implementation - } - @Test public void conditionFutureNeverCompletesWhenNoConditions() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -1835,7 +2554,13 @@ public void conditionFutureNeverCompletesWhenNoConditions() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 1, 2); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(1), + /* recoveryTimeout= */ Duration.ofSeconds(2), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1852,7 +2577,6 @@ public void conditionFutureNeverCompletesWhenNoConditions() throws Exception { @Test public void changeSetAppliedToDataSourceUpdates() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initializerFuture = CompletableFuture.completedFuture( @@ -1865,7 +2589,13 @@ public void changeSetAppliedToDataSourceUpdates() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1877,7 +2607,6 @@ public void changeSetAppliedToDataSourceUpdates() throws Exception { @Test public void multipleChangeSetsAppliedInOrder() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -1891,7 +2620,13 @@ public void multipleChangeSetsAppliedInOrder() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1905,7 +2640,6 @@ public void multipleChangeSetsAppliedInOrder() throws Exception { @Test public void selectorNonEmptyCompletesInitialization() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture firstInitializerFuture = CompletableFuture.completedFuture( @@ -1924,7 +2658,13 @@ public void selectorNonEmptyCompletesInitialization() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, ImmutableList.of(), null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, ImmutableList.of(), null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -1940,7 +2680,6 @@ public void selectorNonEmptyCompletesInitialization() throws Exception { @Test public void initializerChangeSetWithoutSelectorCompletesIfLastInitializer() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initializerFuture = CompletableFuture.completedFuture( @@ -1953,15 +2692,21 @@ public void initializerChangeSetWithoutSelectorCompletesIfLastInitializer() thro ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); startFuture.get(2, TimeUnit.SECONDS); - // A single initializer applied a selectorless basis and there are no synchronizers, - // so once the initializer chain exhausts the applied data is enough to mark the - // data source VALID and complete startFuture. + // A single initializer applied a full payload without a selector and there are no + // synchronizers, so once the initializer chain exhausts the applied data is enough to + // mark the data source VALID and complete startFuture. List statuses = sink.awaitStatuses(1, 2, TimeUnit.SECONDS); assertEquals("Should receive 1 status update", 1, statuses.size()); assertEquals(DataSourceStatusProvider.State.VALID, statuses.get(0)); @@ -1973,7 +2718,6 @@ public void initializerChangeSetWithoutSelectorCompletesIfLastInitializer() thro @Test public void synchronizerChangeSetAlwaysCompletesStartFuture() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -1986,7 +2730,13 @@ public void synchronizerChangeSetAlwaysCompletesStartFuture() throws Exception { () -> new MockSynchronizer(synchronizerFuture) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2001,7 +2751,6 @@ public void synchronizerChangeSetAlwaysCompletesStartFuture() throws Exception { @Test public void goodbyeStatusHandledGracefully() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2015,7 +2764,13 @@ public void goodbyeStatusHandledGracefully() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2030,7 +2785,6 @@ public void goodbyeStatusHandledGracefully() throws Exception { @Test public void shutdownStatusExitsImmediately() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2049,7 +2803,13 @@ public void shutdownStatusExitsImmediately() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2064,7 +2824,6 @@ public void shutdownStatusExitsImmediately() throws Exception { @Test public void fdv1FallbackFlagHonored() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2076,7 +2835,13 @@ public void fdv1FallbackFlagHonored() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2092,7 +2857,6 @@ public void fdv1FallbackFlagHonored() throws Exception { @Test public void emptyInitializerListSkipsToSynchronizers() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2109,7 +2873,13 @@ public void emptyInitializerListSkipsToSynchronizers() throws Exception { } ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture = dataSource.start(); @@ -2121,7 +2891,6 @@ public void emptyInitializerListSkipsToSynchronizers() throws Exception { @Test public void startedFlagPreventsMultipleRuns() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); AtomicInteger runCount = new AtomicInteger(0); @@ -2137,7 +2906,13 @@ public void startedFlagPreventsMultipleRuns() throws Exception { ImmutableList> synchronizers = ImmutableList.of(); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); Future startFuture1 = dataSource.start(); @@ -2156,7 +2931,6 @@ public void startedFlagPreventsMultipleRuns() throws Exception { @Test public void startBeforeRunCompletesAllComplete() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); BlockingQueue syncResults = new LinkedBlockingQueue<>(); @@ -2167,7 +2941,13 @@ public void startBeforeRunCompletesAllComplete() throws Exception { () -> new MockQueuedSynchronizer(syncResults) ); - FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, executor, 120, 300); + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); resourcesToClose.add(dataSource); // Call start multiple times before completion @@ -2187,7 +2967,6 @@ public void startBeforeRunCompletesAllComplete() throws Exception { @Test public void statusTransitionsToValidAfterInitialization() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initializerFuture = CompletableFuture.completedFuture( @@ -2205,9 +2984,12 @@ public void statusTransitionsToValidAfterInitialization() throws Exception { sink, Thread.NORM_PRIORITY, logger, - executor, - 120, - 300 + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -2223,7 +3005,6 @@ public void statusTransitionsToValidAfterInitialization() throws Exception { @Test public void statusIncludesErrorInfoOnFailure() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2251,33 +3032,28 @@ public void statusIncludesErrorInfoOnFailure() throws Exception { sink, Thread.NORM_PRIORITY, logger, - executor, - 120, - 300 + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); - Future startFuture = dataSource.start(); - startFuture.get(2, TimeUnit.SECONDS); - - // Should receive INTERRUPTED first (from terminal error), then OFF (from exhausted synchronizers) - DataSourceStatusProvider.State firstStatus = sink.awaitStatus(2, TimeUnit.SECONDS); - assertNotNull("Should receive first status update", firstStatus); - assertEquals(DataSourceStatusProvider.State.INTERRUPTED, firstStatus); - - DataSourceStatusProvider.State secondStatus = sink.awaitStatus(2, TimeUnit.SECONDS); - assertNotNull("Should receive second status update", secondStatus); - assertEquals(DataSourceStatusProvider.State.OFF, secondStatus); + // Start the data system with a synchronizer that fails with a 401. + dataSource.start(); + DataSourceStatusProvider.State status = sink.awaitStatus(2, TimeUnit.SECONDS); - // Final state should be OFF with error info from the last status update - assertEquals(DataSourceStatusProvider.State.OFF, sink.getLastState()); + // The data system reports INTERRUPTED with the error from the synchronizer. + assertEquals(DataSourceStatusProvider.State.INTERRUPTED, status); assertNotNull("Should have error info", sink.getLastError()); - assertEquals(DataSourceStatusProvider.ErrorKind.UNKNOWN, sink.getLastError().getKind()); + assertEquals(DataSourceStatusProvider.ErrorKind.ERROR_RESPONSE, sink.getLastError().getKind()); + assertEquals(401, sink.getLastError().getStatusCode()); } @Test public void statusRemainsValidDuringSynchronizerOperation() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2299,9 +3075,12 @@ public void statusRemainsValidDuringSynchronizerOperation() throws Exception { sink, Thread.NORM_PRIORITY, logger, - executor, - 120, - 300 + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -2317,16 +3096,15 @@ public void statusRemainsValidDuringSynchronizerOperation() throws Exception { } @Test - public void statusTransitionsFromValidToOffWhenAllSynchronizersFail() throws Exception { - executor = Executors.newScheduledThreadPool(2); + public void statusStaysInterruptedWhileOnlySynchronizerIsParked() throws Exception { MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); - // First synchronizer sends a changeset then terminal error - BlockingQueue syncResults = new LinkedBlockingQueue<>(); - syncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); - syncResults.add(FDv2SourceResult.terminalError( + // The first instance sends a changeset and then a terminal error. Later instances send nothing. + BlockingQueue firstInstanceResults = new LinkedBlockingQueue<>(); + firstInstanceResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + firstInstanceResults.add(FDv2SourceResult.terminalError( new DataSourceStatusProvider.ErrorInfo( DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, @@ -2335,9 +3113,17 @@ public void statusTransitionsFromValidToOffWhenAllSynchronizersFail() throws Exc ), false )); + AtomicInteger buildCount = new AtomicInteger(); + BlockingQueue builds = new LinkedBlockingQueue<>(); ImmutableList> synchronizers = ImmutableList.of( - () -> new MockQueuedSynchronizer(syncResults) + () -> { + int build = buildCount.incrementAndGet(); + builds.offer(build); + return build == 1 + ? new MockQueuedSynchronizer(firstInstanceResults) + : new MockQueuedSynchronizer(new LinkedBlockingQueue<>()); + } ); FDv2DataSource dataSource = new FDv2DataSource( @@ -2347,27 +3133,28 @@ public void statusTransitionsFromValidToOffWhenAllSynchronizersFail() throws Exc sink, Thread.NORM_PRIORITY, logger, - executor, - 120, - 300 + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); - Future startFuture = dataSource.start(); - startFuture.get(2, TimeUnit.SECONDS); - - // Should transition: VALID (from changeset) → INTERRUPTED (from terminal error) → OFF (from exhausted sources) - DataSourceStatusProvider.State firstStatus = sink.awaitStatus(2, TimeUnit.SECONDS); - assertEquals(DataSourceStatusProvider.State.VALID, firstStatus); - - DataSourceStatusProvider.State secondStatus = sink.awaitStatus(2, TimeUnit.SECONDS); - assertEquals(DataSourceStatusProvider.State.INTERRUPTED, secondStatus); - - DataSourceStatusProvider.State thirdStatus = sink.awaitStatus(2, TimeUnit.SECONDS); - assertEquals(DataSourceStatusProvider.State.OFF, thirdStatus); + // Start the data system, and wait until the parked synchronizer is built again. + dataSource.start(); + Integer firstBuild = builds.poll(2, TimeUnit.SECONDS); + Integer rebuild = builds.poll(2, TimeUnit.SECONDS); - assertEquals(DataSourceStatusProvider.State.OFF, sink.getLastState()); - assertNotNull("Should have error info when OFF", sink.getLastError()); + // The status goes from VALID to INTERRUPTED, and stays INTERRUPTED through the park. + assertEquals(Integer.valueOf(1), firstBuild); + assertEquals(Integer.valueOf(2), rebuild); + List statuses = sink.awaitStatuses(2, 2, TimeUnit.SECONDS); + assertEquals(Arrays.asList( + DataSourceStatusProvider.State.VALID, + DataSourceStatusProvider.State.INTERRUPTED), statuses); + assertEquals(DataSourceStatusProvider.State.INTERRUPTED, sink.getLastState()); } // ============================================================================ @@ -2376,7 +3163,6 @@ public void statusTransitionsFromValidToOffWhenAllSynchronizersFail() throws Exc @Test public void fdv1FallbackActivatesWhenFlagSet() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2407,9 +3193,12 @@ public void fdv1FallbackActivatesWhenFlagSet() throws Exception { sink, Thread.NORM_PRIORITY, logger, - executor, - 120, - 300 + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -2425,9 +3214,53 @@ public void fdv1FallbackActivatesWhenFlagSet() throws Exception { assertTrue("Should have at least 2 changesets", sink.getApplyCount() >= 2); } + @Test + public void fdv1FallbackSynchronizerIsParkedAndRestartedDuringDirectedFallback() throws Exception { + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger fdv1BuildCount = new AtomicInteger(); + + // The FDv2 synchronizer reports an interruption that carries the FDv1 fallback directive. + ImmutableList> synchronizers = ImmutableList.of( + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo( + DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), + true)); + return new MockQueuedSynchronizer(results); + } + ); + + // The first instance of the FDv1 fallback synchronizer fails, and the second instance delivers data. + FDv2DataSource.DataSourceFactory fdv1Fallback = () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(fdv1BuildCount.incrementAndGet() == 1 + ? makeTerminalErrorResult() + : FDv2SourceResult.changeSet(makeChangeSet(true), false)); + return new MockQueuedSynchronizer(results); + }; + + // The FDv1 fallback synchronizer is parked for 50-100ms. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, fdv1Fallback, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); + resourcesToClose.add(dataSource); + + // Start the data system. + dataSource.start(); + + // The FDv1 fallback synchronizer is built again after its unpark time, and its data is applied. + sink.awaitApplyCount(1, 2, TimeUnit.SECONDS); + assertEquals(1, sink.getApplyCount()); + assertEquals(2, fdv1BuildCount.get()); + } + @Test public void fdv1FallbackNotCalledWithoutFlag() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2455,9 +3288,12 @@ public void fdv1FallbackNotCalledWithoutFlag() throws Exception { sink, Thread.NORM_PRIORITY, logger, - executor, - 120, - 300 + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -2474,7 +3310,6 @@ public void fdv1FallbackNotCalledWithoutFlag() throws Exception { @Test public void fdv1FallbackWorksAfterInterruption() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2513,9 +3348,12 @@ public void fdv1FallbackWorksAfterInterruption() throws Exception { sink, Thread.NORM_PRIORITY, logger, - executor, - 120, - 300 + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -2531,84 +3369,318 @@ public void fdv1FallbackWorksAfterInterruption() throws Exception { assertTrue("Should have at least 2 changesets", sink.getApplyCount() >= 2); } - // Synchronizer-phase analogue of the initializer halt path: when a synchronizer signals - // FDv1 fallback but no FDv1 fallback synchronizer is configured, the data system must halt - // (transition to OFF, stop building further synchronizers). + // Synchronizer-phase analogue of the initializer recovery path: when a synchronizer signals FDv1 fallback but no + // FDv1 fallback synchronizer is configured, the data system reports INTERRUPTED, runs no synchronizer until the + // fallback TTL ends, and then restarts the FDv2 synchronizers. @Test - public void fdv1FallbackOnSynchronizerWithoutFDv1ConfiguredHaltsDataSystem() throws Exception { - executor = Executors.newScheduledThreadPool(2); + public void fdv1FallbackOnSynchronizerWithoutFDv1ConfiguredRecoversAfterTTL() throws Exception { MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); - - ImmutableList> initializers = ImmutableList.of(); - - // First synchronizer sends a normal payload, then a payload with the directive. - BlockingQueue firstSyncResults = new LinkedBlockingQueue<>(); - firstSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); - firstSyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(false), true)); - - AtomicInteger firstSyncBuildCount = new AtomicInteger(0); + AtomicInteger firstSyncBuildCount = new AtomicInteger(); AtomicBoolean secondSyncCalled = new AtomicBoolean(false); - // Two synchronizers configured; once the directive halts the data system, neither - // should be re-built. The second synchronizer in particular must never run. ImmutableList> synchronizers = ImmutableList.of( + // The first instance sends a payload, then one with the directive. The second sends a payload. () -> { - firstSyncBuildCount.incrementAndGet(); - return new MockQueuedSynchronizer(firstSyncResults); + BlockingQueue results = new LinkedBlockingQueue<>(); + if (firstSyncBuildCount.incrementAndGet() == 1) { + results.add(FDv2SourceResult.changeSet(makeChangeSet(false), false)); + results.add(FDv2SourceResult.changeSet(makeChangeSet(false), true)); + } else { + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + } + return new MockQueuedSynchronizer(results); }, + // The second synchronizer must never run. () -> { secondSyncCalled.set(true); return new MockQueuedSynchronizer(new LinkedBlockingQueue<>()); } ); - // No FDv1 fallback configured. + // No FDv1 fallback synchronizer is configured, and the fallback TTL ends after 50-100ms. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofMillis(100)); + resourcesToClose.add(dataSource); + + // Start the data system. + dataSource.start().get(2, TimeUnit.SECONDS); + + // The data system reports INTERRUPTED, and restarts the first synchronizer after the TTL. + List statuses = sink.awaitStatuses(3, 2, TimeUnit.SECONDS); + assertEquals(Arrays.asList( + DataSourceStatusProvider.State.VALID, + DataSourceStatusProvider.State.INTERRUPTED, + DataSourceStatusProvider.State.VALID), statuses); + assertEquals(2, firstSyncBuildCount.get()); + assertFalse(secondSyncCalled.get()); + } + + // While no FDv1 fallback synchronizer is configured, the FDv2 synchronizer that sent the directive is stopped. + @Test + public void directiveWithoutFDv1ClosesFDv2SynchronizerDuringFallback() throws Exception { + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + CountDownLatch closed = new CountDownLatch(1); + + // The synchronizer sends a payload with the directive, and records when it is closed. + ImmutableList> synchronizers = ImmutableList.of( + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), true)); + return new MockQueuedSynchronizer(results) { + @Override + public void close() { + closed.countDown(); + super.close(); + } + }; + } + ); + + // No FDv1 fallback synchronizer is configured, and the fallback lasts much longer than the test. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1)); + resourcesToClose.add(dataSource); + + // Start the data system. + dataSource.start(); + + // The synchronizer is closed during the fallback. + assertTrue(closed.await(1, TimeUnit.SECONDS)); + } + + // When the fallback TTL ends, the data system stops the FDv1 synchronizer and restarts the FDv2 synchronizers. + @Test + public void fdv1FallbackEndsAfterTTL() throws Exception { + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger fdv2BuildCount = new AtomicInteger(); + AtomicBoolean fdv1Closed = new AtomicBoolean(false); + + ImmutableList> synchronizers = ImmutableList.of( + // The first instance sends a payload with the directive. The second sends a payload. + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), fdv2BuildCount.incrementAndGet() == 1)); + return new MockQueuedSynchronizer(results); + } + ); + + // The FDv1 fallback synchronizer sends a payload, and records when it is closed. + FDv2DataSource.DataSourceFactory fdv1Fallback = () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + return new MockQueuedSynchronizer(results) { + @Override + public void close() { + fdv1Closed.set(true); + super.close(); + } + }; + }; + + // The fallback TTL ends after 50-100ms. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, fdv1Fallback, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofMillis(100)); + resourcesToClose.add(dataSource); + + // Start the data system. + dataSource.start(); + + // After the TTL, the data system has stopped the FDv1 fallback synchronizer and applied FDv2 data again. + sink.awaitApplyCount(3, 2, TimeUnit.SECONDS); + assertEquals(2, fdv2BuildCount.get()); + assertTrue(fdv1Closed.get()); + } + + // When the fallback TTL ends before the FDv1 synchronizer's unpark time, the FDv2 synchronizers restart at the TTL. + @Test + public void fdv1FallbackEndsWhileFDv1SynchronizerIsParked() throws Exception { + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger fdv2BuildCount = new AtomicInteger(); + AtomicInteger fdv1BuildCount = new AtomicInteger(); + + // The first instance reports an interruption with the directive. The second sends a payload with the directive. + ImmutableList> synchronizers = ImmutableList.of( + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + if (fdv2BuildCount.incrementAndGet() == 1) { + results.add(FDv2SourceResult.interrupted( + new DataSourceStatusProvider.ErrorInfo( + DataSourceStatusProvider.ErrorKind.NETWORK_ERROR, 500, null, Instant.now()), + true)); + } else { + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), true)); + } + return new MockQueuedSynchronizer(results); + } + ); + + // The FDv1 fallback synchronizer fails, so the data system parks it. + FDv2DataSource.DataSourceFactory fdv1Fallback = () -> { + fdv1BuildCount.incrementAndGet(); + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(makeTerminalErrorResult()); + return new MockQueuedSynchronizer(results); + }; + + // The FDv1 fallback synchronizer is parked for minutes, and the fallback TTL ends after 100-200ms. FDv2DataSource dataSource = new FDv2DataSource( - initializers, + ImmutableList.of(), synchronizers, - null, + fdv1Fallback, sink, Thread.NORM_PRIORITY, logger, - executor, - 120, - 300 + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofMillis(200) ); resourcesToClose.add(dataSource); - dataSource.start().get(2, TimeUnit.SECONDS); + // Start the data system. + dataSource.start(); - // First payload is applied; the directive-bearing payload is also applied (the SDK - // surfaces the data) but no further synchronizers are dialled. - sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); + // The FDv2 synchronizer is built again after the TTL, and its data is applied. + sink.awaitApplyCount(1, 2, TimeUnit.SECONDS); + assertEquals(1, sink.getApplyCount()); + assertEquals(2, fdv2BuildCount.get()); + + // Each directive falls back to the FDv1 synchronizer, which is still parked, so nothing runs until the TTL. + List statuses = sink.awaitStatuses(4, 2, TimeUnit.SECONDS); + assertEquals(Arrays.asList( + DataSourceStatusProvider.State.INTERRUPTED, + DataSourceStatusProvider.State.INTERRUPTED, + DataSourceStatusProvider.State.VALID, + DataSourceStatusProvider.State.INTERRUPTED), statuses); + assertEquals(1, fdv1BuildCount.get()); + } - // Status must end up OFF because the directive could not be satisfied. - // Poll briefly for the OFF state in case the final transition is observed slightly - // after the apply count reaches 2; this keeps the assertion deterministic without - // relying on real-time delays. - DataSourceStatusProvider.State finalState = null; - for (int i = 0; i < 20 && finalState != DataSourceStatusProvider.State.OFF; i++) { - DataSourceStatusProvider.State next = sink.awaitStatus(100, TimeUnit.MILLISECONDS); - if (next != null) { - finalState = next; - } else if (sink.getLastState() == DataSourceStatusProvider.State.OFF) { - finalState = DataSourceStatusProvider.State.OFF; + // A directive during the FDv1 fallback replaces the TTL, and the FDv1 synchronizer keeps running. + @Test + public void directiveDuringFDv1FallbackExtendsTTL() throws Exception { + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger fdv2BuildCount = new AtomicInteger(); + AtomicInteger fdv1BuildCount = new AtomicInteger(); + BlockingQueue fdv2RestartTimes = new LinkedBlockingQueue<>(); + IterableAsyncQueue fdv1Results = new IterableAsyncQueue<>(); + + // The first instance sends a payload with the directive. The second records when it is built. + ImmutableList> synchronizers = ImmutableList.of( + () -> { + BlockingQueue results = new LinkedBlockingQueue<>(); + if (fdv2BuildCount.incrementAndGet() == 1) { + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), true)); + } else { + fdv2RestartTimes.offer(System.nanoTime()); + } + return new MockQueuedSynchronizer(results); } + ); + + // The FDv1 fallback synchronizer sends the results that the test puts in its queue. + FDv2DataSource.DataSourceFactory fdv1Fallback = () -> { + fdv1BuildCount.incrementAndGet(); + return new MockQueuedSynchronizer(fdv1Results); + }; + + // The fallback TTL ends after 100-200ms. + FDv2DataSource dataSource = new FDv2DataSource( + ImmutableList.of(), + synchronizers, + fdv1Fallback, + sink, + Thread.NORM_PRIORITY, + logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofMillis(200) + ); + resourcesToClose.add(dataSource); + + // Start the data system. FDv1 sends a directive every 50ms until 300ms, which is longer than the TTL. + long start = System.nanoTime(); + dataSource.start(); + for (int delay = 50; delay <= 300; delay += 50) { + executor.schedule( + () -> fdv1Results.put(FDv2SourceResult.changeSet(makeChangeSet(false), true)), + delay, + TimeUnit.MILLISECONDS); } - assertEquals("Status should be OFF after directive halts the data system", - DataSourceStatusProvider.State.OFF, sink.getLastState()); - // The second synchronizer must never be built. - assertFalse("Subsequent synchronizers must not run after directive-induced halt", - secondSyncCalled.get()); - // The first synchronizer should also not be re-built (we should stay halted). - assertEquals("First synchronizer should be built exactly once", - 1, firstSyncBuildCount.get()); + // FDv2 restarts at least the shortest TTL after the last directive, and FDv1 never restarts. + Long fdv2Restart = fdv2RestartTimes.poll(2, TimeUnit.SECONDS); + assertNotNull(fdv2Restart); + long restartMillis = TimeUnit.NANOSECONDS.toMillis(fdv2Restart - start); + assertTrue("restarted FDv2 after " + restartMillis + "ms", restartMillis >= 400); + assertEquals(1, fdv1BuildCount.get()); + } + + // When the FDv2 synchronizers send the directive again after the fallback TTL ends, the data system falls back again. + @Test + public void directiveDuringRecoveryFallsBackToFDv1Again() throws Exception { + MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger fdv2BuildCount = new AtomicInteger(); + AtomicInteger fdv1BuildCount = new AtomicInteger(); + + // Every instance of the FDv2 synchronizer sends a payload with the directive. + ImmutableList> synchronizers = ImmutableList.of( + () -> { + fdv2BuildCount.incrementAndGet(); + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), true)); + return new MockQueuedSynchronizer(results); + } + ); + + // Every instance of the FDv1 fallback synchronizer sends a payload. + FDv2DataSource.DataSourceFactory fdv1Fallback = () -> { + fdv1BuildCount.incrementAndGet(); + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + return new MockQueuedSynchronizer(results); + }; + + // The fallback TTL ends after 50-100ms. + FDv2DataSource dataSource = new FDv2DataSource(ImmutableList.of(), synchronizers, fdv1Fallback, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofMillis(100)); + resourcesToClose.add(dataSource); + + // Start the data system. + dataSource.start(); + + // After the TTL, the FDv2 synchronizer sends the directive again, and the data system starts FDv1 again. + sink.awaitApplyCount(4, 2, TimeUnit.SECONDS); + assertEquals(2, fdv2BuildCount.get()); + assertEquals(2, fdv1BuildCount.get()); } @Test public void fdv1FallbackBlocksOtherSynchronizers() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2646,9 +3718,12 @@ public void fdv1FallbackBlocksOtherSynchronizers() throws Exception { sink, Thread.NORM_PRIORITY, logger, - executor, - 120, - 300 + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -2666,7 +3741,6 @@ public void fdv1FallbackBlocksOtherSynchronizers() throws Exception { @Test public void fdv1FallbackOnlyCalledOncePerDataSource() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); ImmutableList> initializers = ImmutableList.of(); @@ -2698,9 +3772,12 @@ public void fdv1FallbackOnlyCalledOncePerDataSource() throws Exception { sink, Thread.NORM_PRIORITY, logger, - executor, - 120, - 300 + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -2725,7 +3802,6 @@ public void fdv1FallbackOnlyCalledOncePerDataSource() throws Exception { // without giving any of the configured FDv2 synchronizers a chance to run. @Test public void fdv1FallbackOnInitializerSuccessAppliesPayloadAndSwitches() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); // Initializer returns a payload with a selector AND the FDv1 fallback signal. @@ -2760,9 +3836,12 @@ public void fdv1FallbackOnInitializerSuccessAppliesPayloadAndSwitches() throws E sink, Thread.NORM_PRIORITY, logger, - executor, - 120, - 300 + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -2785,7 +3864,6 @@ public void fdv1FallbackOnInitializerSuccessAppliesPayloadAndSwitches() throws E // to the FDv1 fallback synchronizer without trying FDv2 synchronizers. @Test public void fdv1FallbackOnInitializerErrorSwitchesToFDv1() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initFuture = CompletableFuture.completedFuture( @@ -2827,9 +3905,12 @@ public void fdv1FallbackOnInitializerErrorSwitchesToFDv1() throws Exception { sink, Thread.NORM_PRIORITY, logger, - executor, - 120, - 300 + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); @@ -2842,14 +3923,14 @@ public void fdv1FallbackOnInitializerErrorSwitchesToFDv1() throws Exception { fdv2SyncCalled.get()); } - // When an initializer signals FDv1 fallback but no FDv1 synchronizer is configured, the - // SDK must transition the data source status to OFF -- not stay stuck at INITIALIZING -- - // and surface the underlying initializer error so monitors can see why. + // When an initializer signals FDv1 fallback but no FDv1 synchronizer is configured, the data system reports + // INTERRUPTED, runs no synchronizer until the fallback TTL ends, and then starts the FDv2 synchronizers. @Test - public void fdv1FallbackOnInitializerWithoutFDv1ConfiguredTransitionsToOff() throws Exception { - executor = Executors.newScheduledThreadPool(1); + public void fdv1FallbackOnInitializerWithoutFDv1ConfiguredRecoversAfterTTL() throws Exception { MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); + AtomicInteger syncBuildCount = new AtomicInteger(); + // The initializer fails with the directive. DataSourceStatusProvider.ErrorInfo initError = new DataSourceStatusProvider.ErrorInfo( DataSourceStatusProvider.ErrorKind.ERROR_RESPONSE, 500, @@ -2863,49 +3944,44 @@ public void fdv1FallbackOnInitializerWithoutFDv1ConfiguredTransitionsToOff() thr () -> new MockInitializer(initFuture) ); - AtomicBoolean fdv2SyncCalled = new AtomicBoolean(false); + // The synchronizer delivers data. ImmutableList> synchronizers = ImmutableList.of( () -> { - fdv2SyncCalled.set(true); - return new MockQueuedSynchronizer(new LinkedBlockingQueue<>()); + syncBuildCount.incrementAndGet(); + BlockingQueue results = new LinkedBlockingQueue<>(); + results.add(FDv2SourceResult.changeSet(makeChangeSet(true), false)); + return new MockQueuedSynchronizer(results); } ); - // No FDv1 fallback configured. - FDv2DataSource dataSource = new FDv2DataSource( - initializers, - synchronizers, - null, - sink, - Thread.NORM_PRIORITY, - logger, - executor, - 120, - 300 - ); + // No FDv1 fallback synchronizer is configured, and the fallback TTL ends after 50-100ms. + FDv2DataSource dataSource = new FDv2DataSource(initializers, synchronizers, null, sink, Thread.NORM_PRIORITY, logger, + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofMillis(100)); resourcesToClose.add(dataSource); + // Start the data system, and wait for it to be initialized by the synchronizer. dataSource.start().get(2, TimeUnit.SECONDS); - // Status must end up OFF (not INITIALIZING) so callers can observe the terminal state. - assertEquals(DataSourceStatusProvider.State.OFF, sink.getLastState()); - assertNotNull("Initializer error should be preserved on the OFF status", sink.getLastError()); - assertEquals(initError.getKind(), sink.getLastError().getKind()); - assertEquals(initError.getStatusCode(), sink.getLastError().getStatusCode()); - - // FDv2 synchronizers should not run. - assertFalse(fdv2SyncCalled.get()); + // The data system never reported OFF, and started the synchronizer after the TTL. + List statuses = sink.awaitStatuses(3, 2, TimeUnit.SECONDS); + assertFalse(statuses.contains(DataSourceStatusProvider.State.OFF)); + assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); + assertEquals(1, syncBuildCount.get()); } - // When an initializer returns a successful payload-without-selector AND the FDv1 fallback - // flag, the partial payload should still be applied (so evaluations can serve it) and the + // When an initializer returns a full payload without a selector AND the FDv1 fallback + // flag, the payload should still be applied (so evaluations can serve it) and the // SDK should move on to the FDv1 synchronizer rather than scanning further FDv2 sources. - // Crucially, a selectorless basis must NOT mark the data source VALID -- that transition - // belongs to the FDv1 synchronizer once it serves a selectorful payload. Match Go/Python/ + // A full payload without a selector is not a basis, so it does not raise the readiness + // signal -- the FDv1 synchronizer raises it when it serves a full payload. Match Go/Python/ // Ruby behavior on initialization gating. @Test public void fdv1FallbackOnInitializerSuccessNoSelectorAppliesPayloadAndSwitches() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initFuture = CompletableFuture.completedFuture( @@ -2917,7 +3993,7 @@ public void fdv1FallbackOnInitializerSuccessNoSelectorAppliesPayloadAndSwitches( ImmutableList> synchronizers = ImmutableList.of(); - // FDv1 sync must serve a selectorful basis to move the data source to VALID. + // The FDv1 synchronizer serves a full payload, which raises the readiness signal. BlockingQueue fdv1SyncResults = new LinkedBlockingQueue<>(); fdv1SyncResults.add(FDv2SourceResult.changeSet(makeChangeSet(true), false)); @@ -2934,30 +4010,31 @@ public void fdv1FallbackOnInitializerSuccessNoSelectorAppliesPayloadAndSwitches( sink, Thread.NORM_PRIORITY, logger, - executor, - 120, - 300 + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); dataSource.start().get(2, TimeUnit.SECONDS); - // Both payloads applied: the selectorless initializer basis and the selectorful FDv1 basis. + // Both payloads applied: the initializer's payload without a selector and the FDv1 payload. sink.awaitApplyCount(2, 2, TimeUnit.SECONDS); assertTrue("Both initializer and FDv1 payloads should have been applied", sink.getApplyCount() >= 2); assertNotNull(fdv1CalledQueue.poll(2, TimeUnit.SECONDS)); - // VALID may only come from the FDv1 synchronizer's selectorful basis. The data source - // is initialized iff that transition has been observed. + // The FDv1 synchronizer's full payload raised the readiness signal. assertEquals(DataSourceStatusProvider.State.VALID, sink.getLastState()); - assertTrue("isInitialized() must be true after FDv1 served selectorful basis", + assertTrue("isInitialized() must be true after FDv1 served a full payload", dataSource.isInitialized()); } @Test public void orchestrationLogging_warnsWhenNoInitializersOrSynchronizersConfigured() throws Exception { - executor = Executors.newScheduledThreadPool(1); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); FDv2DataSource dataSource = new FDv2DataSource( ImmutableList.of(), @@ -2966,9 +4043,12 @@ public void orchestrationLogging_warnsWhenNoInitializersOrSynchronizersConfigure sink, Thread.NORM_PRIORITY, testLogger, - executor, - 120, - 300 + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); dataSource.start().get(2, TimeUnit.SECONDS); @@ -2979,7 +4059,6 @@ public void orchestrationLogging_warnsWhenNoInitializersOrSynchronizersConfigure @Test public void orchestrationLogging_logsInitializerStartSuccessAndErrorOnException() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture first = new CompletableFuture<>(); first.completeExceptionally(new RuntimeException("first-init-failure")); @@ -2997,9 +4076,12 @@ public void orchestrationLogging_logsInitializerStartSuccessAndErrorOnException( sink, Thread.NORM_PRIORITY, testLogger, - executor, - 120, - 300 + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); dataSource.start().get(2, TimeUnit.SECONDS); @@ -3011,7 +4093,6 @@ public void orchestrationLogging_logsInitializerStartSuccessAndErrorOnException( @Test public void orchestrationLogging_logsInitializerFailedOnStatusResult() throws Exception { - executor = Executors.newScheduledThreadPool(1); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); CompletableFuture initFuture = CompletableFuture.completedFuture( FDv2SourceResult.interrupted( @@ -3034,9 +4115,12 @@ public void orchestrationLogging_logsInitializerFailedOnStatusResult() throws Ex sink, Thread.NORM_PRIORITY, testLogger, - executor, - 120, - 300 + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); dataSource.start().get(2, TimeUnit.SECONDS); @@ -3047,7 +4131,6 @@ public void orchestrationLogging_logsInitializerFailedOnStatusResult() throws Ex @Test public void orchestrationLogging_logsSynchronizerStartingStatusInterruptedAndShutdown() throws Exception { - executor = Executors.newScheduledThreadPool(2); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); IterableAsyncQueue q = new IterableAsyncQueue<>(); q.put(FDv2SourceResult.interrupted( @@ -3070,25 +4153,34 @@ public void orchestrationLogging_logsSynchronizerStartingStatusInterruptedAndShu sink, Thread.NORM_PRIORITY, testLogger, - executor, - 120, - 300 + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofSeconds(1), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); - dataSource.start().get(2, TimeUnit.SECONDS); + dataSource.start(); + // Wait for the status from the interruption, and the status from parking the synchronizer after it shuts down. + sink.awaitStatuses(2, 2, TimeUnit.SECONDS); assertTrue(logTextContains(logCapture, LDLogLevel.INFO, "Synchronizer 'primary-sync' is starting.")); assertTrue(logTextContains(logCapture, LDLogLevel.INFO, "Synchronizer 'primary-sync' reported status: INTERRUPTED.")); assertTrue(logTextContains(logCapture, LDLogLevel.INFO, "Synchronizer 'primary-sync' reported status: SHUTDOWN.")); } @Test - public void orchestrationLogging_logsSynchronizerPermanentFailureAndNoMoreSynchronizers() throws Exception { - executor = Executors.newScheduledThreadPool(1); + public void orchestrationLogging_logsSynchronizerParkedAndAllParked() throws Exception { MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); - IterableAsyncQueue q = new IterableAsyncQueue<>(); - q.put(makeTerminalErrorResult()); + BlockingQueue builds = new LinkedBlockingQueue<>(); + // Every instance of the only synchronizer fails with a terminal error. Each build is recorded. ImmutableList> synchronizers = ImmutableList.of( - () -> new MockQueuedSynchronizer(q, "failing-sync") + () -> { + builds.offer(1); + IterableAsyncQueue q = new IterableAsyncQueue<>(); + q.put(makeTerminalErrorResult()); + return new MockQueuedSynchronizer(q, "failing-sync"); + } ); FDv2DataSource dataSource = new FDv2DataSource( ImmutableList.of(), @@ -3097,33 +4189,36 @@ public void orchestrationLogging_logsSynchronizerPermanentFailureAndNoMoreSynchr sink, Thread.NORM_PRIORITY, testLogger, - executor, - 120, - 300 + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMillis(100), + /* parkMaxDelay= */ Duration.ofSeconds(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); - dataSource.start().get(2, TimeUnit.SECONDS); + + // Start the data system, and wait until the parked synchronizer is built again. + dataSource.start(); + builds.poll(2, TimeUnit.SECONDS); + Integer rebuild = builds.poll(2, TimeUnit.SECONDS); + + // The data system logs the park, and logs that every synchronizer is parked. + assertNotNull(rebuild); assertTrue(logTextContains(logCapture, LDLogLevel.INFO, "Synchronizer 'failing-sync' is starting.")); assertTrue(logTextContains(logCapture, LDLogLevel.INFO, "Synchronizer 'failing-sync' reported status: TERMINAL_ERROR.")); - assertTrue( - logTextContains( - logCapture, - LDLogLevel.WARN, - "Synchronizer 'failing-sync' permanently failed and will not be used again until application restart." - ) - ); - assertTrue(logTextContains(logCapture, LDLogLevel.WARN, "No more synchronizers available.")); + assertTrue(logTextContains(logCapture, LDLogLevel.WARN, "Synchronizer 'failing-sync' is backing off for ")); + assertTrue(logTextContains(logCapture, LDLogLevel.WARN, "All synchronizers are backing off for at least ")); } @Test public void orchestrationLogging_logsErrorWhenSynchronizerNextThrows() throws Exception { - executor = Executors.newScheduledThreadPool(1); MockDataSourceUpdateSink sink = new MockDataSourceUpdateSink(); - // Two tiers: the first throws on next() once; the second shuts down immediately so the outer loop - // does not spin forever re-building the same throwing factory (single-tier would never block). + // The second synchronizer delivers data, so start() completes. IterableAsyncQueue followUpQueue = new IterableAsyncQueue<>(); - followUpQueue.put(FDv2SourceResult.shutdown()); + followUpQueue.put(FDv2SourceResult.changeSet(makeChangeSet(true), false)); ImmutableList> synchronizers = ImmutableList.of( + // The first synchronizer fails from next(). () -> new MockSynchronizer(() -> { throw new RuntimeException("next-boom"); }, "throwing-sync"), @@ -3136,9 +4231,12 @@ public void orchestrationLogging_logsErrorWhenSynchronizerNextThrows() throws Ex sink, Thread.NORM_PRIORITY, testLogger, - executor, - 120, - 300 + /* fallbackTimeout= */ Duration.ofSeconds(120), + /* recoveryTimeout= */ Duration.ofSeconds(300), + /* parkInitialDelay= */ Duration.ofMinutes(5), + /* parkMaxDelay= */ Duration.ofHours(1), + /* failoverDelay= */ Duration.ofMillis(10), + /* defaultFDv1FallbackTTL= */ Duration.ofHours(1) ); resourcesToClose.add(dataSource); dataSource.start().get(2, TimeUnit.SECONDS); @@ -3169,9 +4267,12 @@ public boolean apply(ChangeSet afterFailure = synchronizer.next(); + FDv2SourceResult unexpected = null; + try { + unexpected = afterFailure.get(2, TimeUnit.SECONDS); + } catch (TimeoutException e) { + // No result arrived, which is what the test expects. + } + + // The failure cancels the timer, so the stream does not report that the reset threshold was met. + assertEquals(SourceResultType.CHANGE_SET, changeSet.getResultType()); + assertEquals(SourceSignal.INTERRUPTED, failure.getStatus().getState()); + assertNull(unexpected); + + synchronizer.close(); + } + } + + @Test + public void certificateFailureIsTerminal() throws Exception { + ServerTLSConfiguration untrustedCertificate = ServerTLSConfiguration.makeSelfSignedCertificate(); + + try (HttpServer server = HttpServer.startSecure(untrustedCertificate, Handlers.SSE.start())) { + HttpProperties httpProperties = toHttpProperties(clientContext("sdk-key", baseConfig().build()).getHttp()); + + StreamingSynchronizerImpl synchronizer = new StreamingSynchronizerImpl( + httpProperties, + server.getUri(), + "/stream", + testLogger, + mockSelectorSource(), + executor, + null, + Duration.ofMillis(100), + StreamingSynchronizerImpl.RESET_THRESHOLD, + Thread.NORM_PRIORITY, + null + ); + + // Connect to a server whose certificate the client does not trust. + FDv2SourceResult result = synchronizer.next().get(5, TimeUnit.SECONDS); + + // A certificate failure is terminal for this synchronizer. + assertEquals(SourceResultType.STATUS, result.getResultType()); + assertEquals(SourceSignal.TERMINAL_ERROR, result.getStatus().getState()); + + synchronizer.close(); + } + } + + @Test + public void unexpectedExceptionOnStreamThreadIsTerminal() throws Exception { + try (HttpServer server = HttpServer.start(Handlers.SSE.start())) { + HttpProperties httpProperties = toHttpProperties(clientContext("sdk-key", baseConfig().build()).getHttp()); + // The selector source throws when the stream connects, which ends the stream thread. + SelectorSource selectorSource = mock(SelectorSource.class); + when(selectorSource.getSelector()).thenThrow(new IllegalStateException("unexpected")); + + StreamingSynchronizerImpl synchronizer = new StreamingSynchronizerImpl( + httpProperties, + server.getUri(), + "/stream", + testLogger, + selectorSource, + executor, + null, + Duration.ofMillis(100), + StreamingSynchronizerImpl.RESET_THRESHOLD, + Thread.NORM_PRIORITY, + null + ); + + // Start the stream. + FDv2SourceResult result = synchronizer.next().get(5, TimeUnit.SECONDS); + + // The stream thread cannot recover, so the failure is terminal for this synchronizer. + assertEquals(SourceResultType.STATUS, result.getResultType()); + assertEquals(SourceSignal.TERMINAL_ERROR, result.getStatus().getState()); + + synchronizer.close(); + } + } }