From 5a4c0aea69748a19845a97a48a28cc80ede1e464 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Sun, 29 Mar 2026 00:00:55 +0800 Subject: [PATCH 1/5] [kv] Implement KvTablet RocksDB Lazy Open --- .../apache/fluss/config/ConfigOptions.java | 29 +- .../org/apache/fluss/metrics/MetricNames.java | 3 + .../server/kv/KvIdleReleaseController.java | 152 ++ .../org/apache/fluss/server/kv/KvManager.java | 239 ++- .../org/apache/fluss/server/kv/KvTablet.java | 412 +++++- .../server/kv/KvTabletLazyLifecycle.java | 1056 ++++++++++++++ .../group/TabletServerMetricGroup.java | 22 + .../apache/fluss/server/replica/Replica.java | 485 ++++-- .../fluss/server/replica/ReplicaManager.java | 49 + .../kv/KvIdleReleaseControllerTest.java | 331 +++++ .../apache/fluss/server/kv/KvManagerTest.java | 123 ++ .../server/kv/KvTabletLazyLifecycleTest.java | 1295 +++++++++++++++++ .../server/replica/ReplicaLazyOpenTest.java | 410 ++++++ .../fluss/server/replica/ReplicaTest.java | 80 + .../HistoricalPartitionManagerTest.java | 74 +- .../observability/monitor-metrics.md | 16 + 16 files changed, 4630 insertions(+), 146 deletions(-) create mode 100644 fluss-server/src/main/java/org/apache/fluss/server/kv/KvIdleReleaseController.java create mode 100644 fluss-server/src/main/java/org/apache/fluss/server/kv/KvTabletLazyLifecycle.java create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/kv/KvIdleReleaseControllerTest.java create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletLazyLifecycleTest.java create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaLazyOpenTest.java diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index e497ea07039..d88615673db 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -2681,9 +2681,22 @@ public class ConfigOptions { + "so that clients begin throttling before the storage engine is forced to throttle itself. " + "The gap between this value and the storage trigger forms the throttle ramp-up window."); - // ------------------------------------------------------------------------ + // ConfigOptions for KV lazy open + + public static final ConfigOption KV_LAZY_OPEN_ENABLED = + key("kv.lazy-open.enabled") + .booleanType() + .defaultValue(false) + .withDescription("Whether to enable KvTablet lazy open."); + + public static final ConfigOption KV_LAZY_OPEN_IDLE_TIMEOUT = + key("kv.lazy-open.idle-timeout") + .durationType() + .defaultValue(Duration.ofHours(24)) + .withDescription( + "Idle time before an open KvTablet is eligible for release back to lazy state."); + // ConfigOptions for metrics - // ------------------------------------------------------------------------ public static final ConfigOption> METRICS_REPORTERS = key("metrics.reporters") .stringType() @@ -2704,9 +2717,7 @@ public class ConfigOptions { + "the CoordinatorServer) it is advisable to use a port range " + "like 9250-9260."); - // ------------------------------------------------------------------------ // ConfigOptions for prometheus push gateway reporter - // ------------------------------------------------------------------------ public static final ConfigOption METRICS_REPORTER_PROMETHEUS_PUSHGATEWAY_HOST_URL = key("metrics.reporter.prometheus-push.host-url") .stringType() @@ -2771,9 +2782,7 @@ public class ConfigOptions { + "The value is automatically redacted when the configuration " + "is logged or displayed."); - // ------------------------------------------------------------------------ // ConfigOptions for jmx reporter - // ------------------------------------------------------------------------ public static final ConfigOption METRICS_REPORTER_JMX_HOST = key("metrics.reporter.jmx.port") .stringType() @@ -2786,9 +2795,7 @@ public class ConfigOptions { + "the CoordinatorServer) it is advisable to use a port range " + "like 9990-9999."); - // ------------------------------------------------------------------------ // ConfigOptions for influxdb reporter - // ------------------------------------------------------------------------ public static final ConfigOption METRICS_REPORTER_INFLUXDB_VERSION = key("metrics.reporter.influxdb.version") .stringType() @@ -2830,9 +2837,7 @@ public class ConfigOptions { .defaultValue(Duration.ofSeconds(10)) .withDescription("The interval of reporting metrics to InfluxDB."); - // ------------------------------------------------------------------------ // ConfigOptions for lakehouse storage - // ------------------------------------------------------------------------ public static final ConfigOption DATALAKE_ENABLED = key("datalake.enabled") .booleanType() @@ -2853,9 +2858,7 @@ public class ConfigOptions { "The datalake format used by Fluss as lakehouse storage. Currently, supported formats are Paimon, Iceberg, Hudi, and Lance. " + "In the future, more kinds of data lake format will be supported, such as DeltaLake."); - // ------------------------------------------------------------------------ // ConfigOptions for tiering service - // ------------------------------------------------------------------------ public static final ConfigOption LAKE_TIERING_AUTO_EXPIRE_SNAPSHOT = key("lake.tiering.auto-expire-snapshot") @@ -2877,9 +2880,7 @@ public class ConfigOptions { + "If not configured and the tiering service runs in a Flink job, Fluss uses " + "Flink's IO temporary directories with a 'fluss' child directory."); - // ------------------------------------------------------------------------ // ConfigOptions for fluss kafka - // ------------------------------------------------------------------------ public static final ConfigOption KAFKA_ENABLED = key("kafka.enabled") .booleanType() diff --git a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java index d1ab0ae6c90..a9a2f3d8fff 100644 --- a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java +++ b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java @@ -175,6 +175,9 @@ public class MetricNames { "preWriteBufferTruncateAsDuplicatedPerSecond"; public static final String KV_PRE_WRITE_BUFFER_TRUNCATE_AS_ERROR_RATE = "preWriteBufferTruncateAsErrorPerSecond"; + public static final String KV_TABLET_OPEN_COUNT = "kvTabletOpenCount"; + public static final String KV_TABLET_LAZY_COUNT = "kvTabletLazyCount"; + public static final String KV_TABLET_FAILED_COUNT = "kvTabletFailedCount"; // -------------------------------------------------------------------------------------------- // RocksDB metrics diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvIdleReleaseController.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvIdleReleaseController.java new file mode 100644 index 00000000000..e290696f8d8 --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvIdleReleaseController.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.kv; + +import org.apache.fluss.annotation.VisibleForTesting; +import org.apache.fluss.utils.clock.Clock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** + * Periodically checks OPEN KvTablets and releases idle ones back to LAZY state. Tablets are sorted + * by last access time (LRU). Operates directly on {@link KvTablet} — no dependency on Replica + * layer. + */ +public class KvIdleReleaseController implements Closeable { + + private static final Logger LOG = LoggerFactory.getLogger(KvIdleReleaseController.class); + + private final ScheduledExecutorService scheduler; + private final Supplier> tabletSupplier; + private final Clock clock; + + private final long checkIntervalMs; + private final long idleIntervalMs; + + private volatile ScheduledFuture scheduledTask; + + public KvIdleReleaseController( + ScheduledExecutorService scheduler, + Supplier> tabletSupplier, + Clock clock, + long checkIntervalMs, + long idleIntervalMs) { + this.scheduler = scheduler; + this.tabletSupplier = tabletSupplier; + this.clock = clock; + this.checkIntervalMs = checkIntervalMs; + this.idleIntervalMs = idleIntervalMs; + } + + public void start() { + scheduledTask = + scheduler.scheduleWithFixedDelay( + this::checkAndRelease, + checkIntervalMs, + checkIntervalMs, + TimeUnit.MILLISECONDS); + LOG.info( + "KvIdleReleaseController started: checkInterval={}ms, idleInterval={}ms", + checkIntervalMs, + idleIntervalMs); + } + + /** + * Scans all open tablets, selects those idle beyond the configured threshold, and releases them + * in LRU order (coldest first). + */ + @VisibleForTesting + void checkAndRelease() { + try { + Collection tablets = tabletSupplier.get(); + long now = clock.milliseconds(); + + // Snapshot timestamp once per tablet to avoid double-read of volatile field + List idleCandidates = + tablets.stream() + .map(t -> new ReleaseCandidate(t, t.getLastAccessTimestamp())) + .filter(c -> now - c.lastAccessTimestamp > idleIntervalMs) + .sorted(Comparator.comparingLong(c -> c.lastAccessTimestamp)) + .collect(Collectors.toList()); + + if (idleCandidates.isEmpty()) { + LOG.debug("Idle release round: no idle tablets found"); + return; + } + + LOG.info("Idle release round: open={}, idle={}", tablets.size(), idleCandidates.size()); + + int released = 0; + for (ReleaseCandidate candidate : idleCandidates) { + KvTablet tablet = candidate.tablet; + + if (tablet.canRelease(idleIntervalMs, now)) { + try { + boolean success = tablet.releaseKv(); + if (success) { + released++; + LOG.debug( + "Released KvTablet for {} (idle {}ms)", + tablet.getTableBucket(), + now - candidate.lastAccessTimestamp); + } + } catch (Exception e) { + LOG.warn("Failed to release KvTablet for {}", tablet.getTableBucket(), e); + } + } + } + + if (released > 0) { + LOG.info( + "Idle release round complete: released={}/{}", + released, + idleCandidates.size()); + } + } catch (Exception e) { + LOG.error("Error during idle release check", e); + } + } + + @Override + public void close() { + if (scheduledTask != null) { + scheduledTask.cancel(false); + } + } + + private static class ReleaseCandidate { + final KvTablet tablet; + final long lastAccessTimestamp; + + ReleaseCandidate(KvTablet tablet, long lastAccessTimestamp) { + this.tablet = tablet; + this.lastAccessTimestamp = lastAccessTimestamp; + } + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java index 29eedc5eb6c..80ff2c5e490 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java @@ -83,6 +83,7 @@ import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; import java.util.function.Predicate; import static org.apache.fluss.utils.Preconditions.checkState; @@ -129,6 +130,21 @@ public static RateLimiter getDefaultRateLimiter() { return DEFAULT_RATE_LIMITER; } + // ---- KV lazy open internal constants ---- + // Not exposed as ConfigOptions because these are internal tuning parameters that should + // rarely need adjustment. If operational experience shows otherwise, promote to ConfigOptions + // with keys like "kv.lazy-open.max-concurrent-opens", "kv.lazy-open.open-timeout", etc. + private static final int LAZY_OPEN_MAX_CONCURRENT_OPENS = 10; + private static final long LAZY_OPEN_TIMEOUT_MS = 300_000; + private static final long LAZY_OPEN_FAILED_BACKOFF_BASE_MS = 5_000; + private static final long LAZY_OPEN_FAILED_BACKOFF_MAX_MS = 300_000; + + /** + * Timeout for draining pins during release in milliseconds. Public because {@link + * org.apache.fluss.server.replica.Replica} (different package) needs this value. + */ + public static final long RELEASE_DRAIN_TIMEOUT_MS = 5_000; + private final LogManager logManager; private final LocalDiskManager localDiskManager; @@ -174,6 +190,13 @@ public static RateLimiter getDefaultRateLimiter() { private final KvFlushScheduler kvFlushScheduler; + // ---- KV lazy open configuration ---- + private final boolean lazyOpenEnabled; + private final @Nullable Semaphore openSemaphore; + private final long openTimeoutMs; + private final long failedBackoffBaseMs; + private final long failedBackoffMaxMs; + private volatile boolean isShutdown = false; private KvManager( @@ -194,6 +217,19 @@ private KvManager( this.zkClient = zkClient; this.clock = clock; this.remoteKvDir = FlussPaths.remoteKvDir(conf); + // KV lazy open configuration + this.lazyOpenEnabled = conf.get(ConfigOptions.KV_LAZY_OPEN_ENABLED); + if (lazyOpenEnabled) { + this.openSemaphore = new Semaphore(LAZY_OPEN_MAX_CONCURRENT_OPENS); + this.openTimeoutMs = LAZY_OPEN_TIMEOUT_MS; + this.failedBackoffBaseMs = LAZY_OPEN_FAILED_BACKOFF_BASE_MS; + this.failedBackoffMaxMs = LAZY_OPEN_FAILED_BACKOFF_MAX_MS; + } else { + this.openSemaphore = null; + this.openTimeoutMs = 0; + this.failedBackoffBaseMs = 0; + this.failedBackoffMaxMs = 0; + } this.remoteFileSystem = remoteKvDir.getFileSystem(); this.serverMetricGroup = tabletServerMetricGroup; this.sharedRocksDBRateLimiter = createSharedRateLimiter(conf); @@ -347,6 +383,31 @@ public static KvManager create( clock); } + /** Returns whether KV tablets open their RocksDB resources on first access. */ + public boolean isLazyOpenEnabled() { + return lazyOpenEnabled; + } + + /** Returns the semaphore limiting concurrent lazy opens, or null when disabled. */ + public @Nullable Semaphore getOpenSemaphore() { + return openSemaphore; + } + + /** Returns the lazy-open wait timeout in milliseconds. */ + public long getOpenTimeoutMs() { + return openTimeoutMs; + } + + /** Returns the initial retry backoff after a failed lazy open. */ + public long getFailedBackoffBaseMs() { + return failedBackoffBaseMs; + } + + /** Returns the maximum retry backoff after a failed lazy open. */ + public long getFailedBackoffMaxMs() { + return failedBackoffMaxMs; + } + /** * Returns the shared block cache usage in bytes, or 0 if shared cache is disabled. * @@ -549,54 +610,101 @@ public KvTablet getOrCreateKv( return inKvLock( tableBucket, () -> { - if (currentKvs.containsKey(tableBucket)) { - return currentKvs.get(tableBucket); + KvTablet existing = currentKvs.get(tableBucket); + if (existing != null) { + return existing; } - - File tabletDir = - getOrCreateTabletDir(logTablet.getDataDir(), tablePath, tableBucket); - RowMerger merger = RowMerger.create(tableConfig, kvFormat, schemaGetter); - AutoIncrementManager autoIncrementManager = - new AutoIncrementManager( - schemaGetter, - tablePath.getTablePath(), - tableConfig, - new ZkSequenceGeneratorFactory(zkClient)); - KvTablet tablet = - KvTablet.create( + doCreateKv( tablePath, tableBucket, logTablet, - tabletDir, - conf, - serverMetricGroup, - arrowBufferAllocator, - memorySegmentPool, kvFormat, - merger, - arrowCompressionInfo, schemaGetter, - tableConfig.getChangelogImage(), - sharedRocksDBRateLimiter, - sharedBlockCache, - sharedWriteBufferManager, - kvFlushScheduler, - flushCompleteListener, - autoIncrementManager, - clock, - tableConfig); + tableConfig, + arrowCompressionInfo, + flushCompleteListener); currentKvs.put(tableBucket, tablet); - - LOG.info( - "Created kv tablet for bucket {} in dir {}.", - tableBucket, - tabletDir.getAbsolutePath()); - return tablet; }); } + /** Creates a tablet without replacing the registered lazy tablet. */ + public KvTablet createKvTabletUnregistered( + PhysicalTablePath tablePath, + TableBucket tableBucket, + LogTablet logTablet, + KvFormat kvFormat, + SchemaGetter schemaGetter, + TableConfig tableConfig, + ArrowCompressionInfo arrowCompressionInfo, + @Nullable Runnable flushCompleteListener) + throws Exception { + return inKvLock( + tableBucket, + () -> + doCreateKv( + tablePath, + tableBucket, + logTablet, + kvFormat, + schemaGetter, + tableConfig, + arrowCompressionInfo, + flushCompleteListener)); + } + + private KvTablet doCreateKv( + PhysicalTablePath tablePath, + TableBucket tableBucket, + LogTablet logTablet, + KvFormat kvFormat, + SchemaGetter schemaGetter, + TableConfig tableConfig, + ArrowCompressionInfo arrowCompressionInfo, + @Nullable Runnable flushCompleteListener) + throws Exception { + File tabletDir = getOrCreateTabletDir(logTablet.getDataDir(), tablePath, tableBucket); + RowMerger merger = RowMerger.create(tableConfig, kvFormat, schemaGetter); + AutoIncrementManager autoIncrementManager = + new AutoIncrementManager( + schemaGetter, + tablePath.getTablePath(), + tableConfig, + new ZkSequenceGeneratorFactory(zkClient)); + + KvTablet tablet = + KvTablet.create( + tablePath, + tableBucket, + logTablet, + tabletDir, + conf, + serverMetricGroup, + arrowBufferAllocator, + memorySegmentPool, + kvFormat, + merger, + arrowCompressionInfo, + schemaGetter, + tableConfig.getChangelogImage(), + sharedRocksDBRateLimiter, + sharedBlockCache, + sharedWriteBufferManager, + kvFlushScheduler, + flushCompleteListener, + autoIncrementManager, + clock, + tableConfig); + + LOG.info( + "Created kv tablet for bucket {} in dir {}.", + tableBucket, + tabletDir.getAbsolutePath()); + + return tablet; + } + /** * Create the tablet directory for the given table path and table bucket. * @@ -675,7 +783,27 @@ public KvTablet loadKv( physicalTablePath, tableBucket, schemaGetter, - flushCompleteListener)); + flushCompleteListener, + true)); + } + + /** Loads local state without replacing the registered lazy tablet. */ + public KvTablet loadKvUnregistered( + File tabletDir, SchemaGetter schemaGetter, @Nullable Runnable flushCompleteListener) + throws Exception { + Tuple2 pathAndBucket = FlussPaths.parseTabletDir(tabletDir); + PhysicalTablePath physicalTablePath = pathAndBucket.f0; + TableBucket tableBucket = pathAndBucket.f1; + return inKvLock( + tableBucket, + () -> + doLoadKv( + tabletDir, + physicalTablePath, + tableBucket, + schemaGetter, + flushCompleteListener, + false)); } private KvTablet doLoadKv( @@ -683,10 +811,11 @@ private KvTablet doLoadKv( PhysicalTablePath physicalTablePath, TableBucket tableBucket, SchemaGetter schemaGetter, - @Nullable Runnable flushCompleteListener) + @Nullable Runnable flushCompleteListener, + boolean register) throws Exception { KvTablet currentKv = currentKvs.get(tableBucket); - if (currentKv != null) { + if (register && currentKv != null) { throw new IllegalStateException( String.format( "Duplicate kv tablet directories for bucket %s are found in both %s and %s. " @@ -748,11 +877,41 @@ private KvTablet doLoadKv( autoIncrementManager, clock, tableConfig); - currentKvs.put(tableBucket, kvTablet); + if (register) { + currentKvs.put(tableBucket, kvTablet); + } return kvTablet; } + /** Register a KvTablet (e.g. a lazy sentinel) into the {@code currentKvs} registry. */ + public void registerKv(TableBucket tableBucket, KvTablet kvTablet) { + inKvLock( + tableBucket, + () -> { + if (currentKvs.containsKey(tableBucket)) { + throw new IllegalStateException( + "KvTablet already registered for " + tableBucket); + } + currentKvs.put(tableBucket, kvTablet); + return null; + }); + } + + /** Remove a KvTablet from the {@code currentKvs} registry (e.g. on drop). */ + public void unregisterKv(TableBucket tableBucket) { + inKvLock(tableBucket, () -> currentKvs.remove(tableBucket)); + } + + /** + * Get the tablet directory path for the given table path and table bucket. This is a read-only + * accessor — it does not create the directory. + */ + public File getTabletDirPath( + File dataDir, PhysicalTablePath tablePath, TableBucket tableBucket) { + return getTabletDir(dataDir, tablePath, tableBucket); + } + public void deleteRemoteKvSnapshot( PhysicalTablePath physicalTablePath, TableBucket tableBucket) { FsPath remoteKvTabletDir = diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java index e6bea68f7ab..9b9c852f95e 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java @@ -124,7 +124,7 @@ public final class KvTablet { private static final int MAX_FLUSH_RETRY_BACKOFF_SHIFT = 64 - Long.numberOfLeadingZeros(MAX_FLUSH_RETRY_DELAY_MS / MIN_FLUSH_RETRY_DELAY_MS); - private static final long ROW_COUNT_DISABLED = -1; + public static final long ROW_COUNT_DISABLED = -1; // Retain recent historical KV state within this WAL offset distance of lake progress. // TODO: Consider time-based retention after lake coverage is confirmed. @@ -137,11 +137,36 @@ public final class KvTablet { */ private static final int TARGET_ENTRIES_PER_NATIVE_WRITE = 500; + /** + * A guard that prevents the underlying RocksDB from being released while held. Use with + * try-with-resources. In eager mode, this is a no-op. In lazy mode, it pins the RocksDB + * instance and ensures it is open before returning. + */ + public static final class Guard implements AutoCloseable { + private final @Nullable KvTablet tablet; + private boolean released; + + /** No-op guard for eager mode. */ + static final Guard NOOP = new Guard(null); + + Guard(@Nullable KvTablet tablet) { + this.tablet = tablet; + } + + @Override + public void close() { + if (!released && tablet != null) { + released = true; + tablet.releasePin(); + } + } + } + private final PhysicalTablePath physicalPath; private final TableBucket tableBucket; private final boolean historicalPartition; - private final LogTablet logTablet; + final LogTablet logTablet; private final File kvTabletDir; private final long writeBatchSize; @@ -149,7 +174,7 @@ public final class KvTablet { private final KvPreWriteBuffer kvPreWriteBuffer; private final KvStateAccessor kvStateAccessor; private final KvWriteProcessor kvWriteProcessor; - private final TabletServerMetricGroup serverMetricGroup; + final TabletServerMetricGroup serverMetricGroup; private final KvFlushScheduler kvFlushScheduler; private final boolean closeFlushScheduler; @@ -192,6 +217,12 @@ public final class KvTablet { @GuardedBy("kvLock") private volatile boolean isClosed = false; + @Nullable private final KvTabletLazyLifecycle lifecycle; + + // Keep the opened tablet intact: queued flushes and snapshot callbacks retain its locks + // and state until close has drained all RocksDB resource leases. + @Nullable private volatile KvTablet openedTablet; + private KvTablet( PhysicalTablePath physicalPath, TableBucket tableBucket, @@ -219,6 +250,7 @@ private KvTablet( @Nullable RowTtlTimestampProvider rowTtlTimestampProvider, Clock clock, boolean rowTtlEnabled) { + this.lifecycle = null; this.physicalPath = physicalPath; this.tableBucket = tableBucket; this.historicalPartition = @@ -269,6 +301,97 @@ private KvTablet( : 0L; } + private KvTablet( + PhysicalTablePath physicalPath, + TableBucket tableBucket, + LogTablet logTablet, + @Nullable TabletServerMetricGroup serverMetricGroup) { + this.physicalPath = physicalPath; + this.tableBucket = tableBucket; + this.logTablet = logTablet; + this.serverMetricGroup = serverMetricGroup; + this.historicalPartition = + HISTORICAL_PARTITION_VALUE.equals(physicalPath.getPartitionName()); + this.kvTabletDir = null; + this.writeBatchSize = 0; + this.rocksDBKv = null; + this.kvPreWriteBuffer = null; + this.kvStateAccessor = null; + this.kvWriteProcessor = null; + this.kvFlushScheduler = null; + this.closeFlushScheduler = false; + this.kvValueLayout = null; + this.stateValueEncoder = null; + this.historicalCleanupOffset = new AtomicLong(); + this.rowTtlTimestampProvider = null; + this.rowTtlEnabled = false; + this.autoIncrementManager = null; + this.rocksDBStatistics = null; + this.lifecycle = new KvTabletLazyLifecycle(this); + } + + /** Creates a tablet whose RocksDB resources are opened on first access. */ + public static KvTablet createLazySentinel( + PhysicalTablePath physicalPath, + TableBucket tableBucket, + LogTablet logTablet, + @Nullable TabletServerMetricGroup serverMetricGroup) { + return new KvTablet(physicalPath, tableBucket, logTablet, serverMetricGroup); + } + + /** Returns the lazy lifecycle, or null for an eagerly opened tablet. */ + @Nullable + public KvTabletLazyLifecycle getLifecycle() { + return lifecycle; + } + + void installRocksDB(KvTablet source) { + checkState(source.lifecycle == null, "Only an eager tablet can be installed."); + synchronized (historicalCleanupOffset) { + if (historicalPartition) { + source.advanceHistoricalCleanupOffset(historicalCleanupOffset.get()); + } + this.openedTablet = source; + } + } + + void detachRocksDB() { + detachRocksDB(KvCloseMode.PRESERVE_LOCAL_STATE); + } + + void detachRocksDB(KvCloseMode closeMode) { + KvTablet opened = openedTablet; + if (opened != null) { + try { + opened.close(closeMode); + } catch (Exception e) { + throw new KvStorageException("Failed to close KV tablet " + tableBucket, e); + } + openedTablet = null; + } + } + + private KvTablet requireOpenedTablet() { + return checkNotNull(openedTablet, "RocksDB is not open for %s", tableBucket); + } + + long currentRowCount() { + KvTablet opened = openedTablet; + return lifecycle == null || opened == null ? rowCount : opened.rowCount; + } + + boolean hasActiveResourceLeases() { + KvTablet opened = openedTablet; + return opened != null && opened.rocksDBKv.getResourceGuard().getLeaseCount() > 0; + } + + void deleteLocalDirectory() { + File dir = getKvTabletDir(); + if (dir != null) { + FileUtils.deleteDirectoryQuietly(dir); + } + } + /** * Creates a kv tablet with a dedicated {@link KvFlushScheduler} that is closed together with * the tablet. Production code must use {@link #create(PhysicalTablePath, TableBucket, @@ -618,6 +741,10 @@ RowTtlTimestampProvider getRowTtlTimestampProvider() { /** Returns the physical value layout used by this tablet. */ public KvValueLayout getKvValueLayout() { + if (lifecycle != null) { + return requireOpenedTablet().getKvValueLayout(); + } + return kvValueLayout; } @@ -630,10 +757,20 @@ public TablePath getTablePath() { } public long getAutoIncrementCacheSize() { + if (lifecycle != null) { + KvTablet opened = openedTablet; + return opened == null ? 0 : opened.getAutoIncrementCacheSize(); + } + return autoIncrementManager.getAutoIncrementCacheSize(); } public void updateAutoIncrementIDRange(AutoIncIDRange newRange) { + if (lifecycle != null) { + requireOpenedTablet().updateAutoIncrementIDRange(newRange); + return; + } + autoIncrementManager.updateIDRange(newRange); } @@ -643,11 +780,21 @@ public String getPartitionName() { } public File getKvTabletDir() { + if (lifecycle != null) { + KvTablet opened = openedTablet; + return opened == null ? lifecycle.getTabletDir() : opened.getKvTabletDir(); + } + return kvTabletDir; } /** Returns the total size in bytes of the live RocksDB SST files. */ public long liveSstFilesSize() { + if (lifecycle != null) { + KvTablet opened = openedTablet; + return opened == null ? 0L : opened.liveSstFilesSize(); + } + return rocksDBKv.liveSstFilesSize(); } @@ -658,6 +805,11 @@ public long liveSstFilesSize() { */ @Nullable public RocksDBStatistics getRocksDBStatistics() { + if (lifecycle != null) { + KvTablet opened = openedTablet; + return opened == null ? null : opened.getRocksDBStatistics(); + } + return rocksDBStatistics; } @@ -673,6 +825,19 @@ void setRowCount(long rowCount) { // row_count is volatile, so it's safe to read without lock public long getRowCount() { + if (lifecycle != null) { + KvTablet opened = openedTablet; + if (opened != null) { + return opened.getRowCount(); + } + long cached = lifecycle.getCachedRowCount(); + if (cached == ROW_COUNT_DISABLED) { + throw new InvalidTableException( + "Row count is disabled for table " + getTablePath()); + } + return cached; + } + if (rowCount == ROW_COUNT_DISABLED) { if (rowTtlEnabled) { throw new InvalidTableException( @@ -701,6 +866,10 @@ public long getRowCount() { */ @GuardedBy("kvLock") public TabletState getTabletState() { + if (lifecycle != null) { + return requireOpenedTablet().getTabletState(); + } + return new TabletState( flushedLogOffset, rowCount == ROW_COUNT_DISABLED ? null : rowCount, @@ -745,6 +914,9 @@ public LogAppendInfo putAsLeader(KvRecordBatch kvRecords, @Nullable int[] target public LogAppendInfo putAsLeader( KvRecordBatch kvRecords, @Nullable int[] targetColumns, MergeMode mergeMode) throws Exception { + if (lifecycle != null) { + return requireOpenedTablet().putAsLeader(kvRecords, targetColumns, mergeMode); + } checkState( !historicalPartition, "putAsLeader is not supported for historical KV tablet %s", @@ -766,6 +938,16 @@ public LogAppendInfo putHistoricalAsLeader( String originalPartitionName, HistoricalValueLookup historicalValueLookup) throws Exception { + if (lifecycle != null) { + return requireOpenedTablet() + .putHistoricalAsLeader( + kvRecords, + targetColumns, + mergeMode, + originalPartitionName, + historicalValueLookup); + } + checkState(historicalPartition, "%s is not a historical KV tablet", tableBucket); return putAsLeader( kvRecords, @@ -787,6 +969,12 @@ public LocalValueLookupResult probeLocalPreviousValues( MergeMode mergeMode, String originalPartitionName) throws Exception { + if (lifecycle != null) { + return requireOpenedTablet() + .probeLocalPreviousValues( + kvRecords, targetColumns, mergeMode, originalPartitionName); + } + checkState(historicalPartition, "%s is not a historical KV tablet", tableBucket); return inReadLock( kvLock, @@ -858,6 +1046,16 @@ long localLogEndOffset() { } public void requestFlush(long exclusiveUpToLogOffset, FatalErrorHandler fatalErrorHandler) { + if (lifecycle != null) { + KvTablet opened = openedTablet; + if (opened != null) { + // Enqueue flushes even while release drains an existing writer. The opened + // tablet serializes requests with close and ignores requests after closing. + opened.requestFlush(exclusiveUpToLogOffset, fatalErrorHandler); + } + return; + } + asyncFatalErrorHandler = fatalErrorHandler; inWriteLock(kvLock, () -> requestFlushInternal(exclusiveUpToLogOffset)); } @@ -869,11 +1067,30 @@ public void setFlushCompleteListener(@Nullable Runnable flushCompleteListener) { } public long getFlushedLogOffset() { + if (lifecycle != null) { + KvTablet opened = openedTablet; + return opened == null + ? lifecycle.getCachedFlushedLogOffset() + : opened.getFlushedLogOffset(); + } + return flushedLogOffset; } /** Advances the exclusive historical cleanup offset without allowing it to move backwards. */ public boolean advanceHistoricalCleanupOffset(long cleanupOffset) { + if (lifecycle != null) { + checkState(historicalPartition, "%s is not a historical KV tablet", tableBucket); + checkArgument(cleanupOffset >= 0L, "Historical cleanup offset must be non-negative."); + synchronized (historicalCleanupOffset) { + long previous = historicalCleanupOffset.getAndAccumulate(cleanupOffset, Math::max); + KvTablet opened = openedTablet; + return opened == null + ? cleanupOffset > previous + : opened.advanceHistoricalCleanupOffset(cleanupOffset); + } + } + checkState(historicalPartition, "%s is not a historical KV tablet", tableBucket); checkArgument(cleanupOffset >= 0L, "Historical cleanup offset must be non-negative."); long previousCleanupOffset = @@ -883,6 +1100,13 @@ public boolean advanceHistoricalCleanupOffset(long cleanupOffset) { /** Returns the current exclusive cleanup offset for a historical overlay. */ public long getHistoricalCleanupOffset() { + if (lifecycle != null) { + KvTablet opened = openedTablet; + return opened == null + ? historicalCleanupOffset.get() + : opened.getHistoricalCleanupOffset(); + } + checkState(historicalPartition, "%s is not a historical KV tablet", tableBucket); return historicalCleanupOffset.get(); } @@ -1198,10 +1422,28 @@ void putToPreWriteBuffer( * tablet. */ public Executor getGuardedExecutor() { + if (lifecycle != null) { + KvTablet opened = openedTablet; + return runnable -> { + Guard guard = tryAcquireExistingGuard(); + if (guard != null) { + try (Guard ignored = guard) { + if (opened != null && openedTablet == opened) { + opened.getGuardedExecutor().execute(runnable); + } + } + } + }; + } + return runnable -> inWriteLock(kvLock, runnable::run); } public List multiGet(List keys) throws IOException { + if (lifecycle != null) { + return requireOpenedTablet().multiGet(keys); + } + return inReadLock( kvLock, () -> { @@ -1219,6 +1461,10 @@ public List multiGet(List keys) throws IOException { * #multiGet} so that clients only observe flushed data. */ public List multiGetFromBufferOrKv(List keys) throws IOException { + if (lifecycle != null) { + return requireOpenedTablet().multiGetFromBufferOrKv(keys); + } + checkState( !historicalPartition, "multiGetFromBufferOrKv is not supported for historical KV tablet %s", @@ -1240,6 +1486,10 @@ public List multiGetFromBufferOrKv(List keys) throws IOE /** Looks up one key from the flushed local state for an original historical partition. */ public KvStateLookupResult lookupHistoricalLocal(String originalPartitionName, byte[] key) throws IOException { + if (lifecycle != null) { + return requireOpenedTablet().lookupHistoricalLocal(originalPartitionName, key); + } + checkState(historicalPartition, "%s is not a historical KV tablet", tableBucket); return inReadLock( kvLock, @@ -1258,6 +1508,10 @@ public KvStateLookupResult lookupHistoricalLocal(String originalPartitionName, b } public List prefixLookup(byte[] prefixKey) throws IOException { + if (lifecycle != null) { + return requireOpenedTablet().prefixLookup(prefixKey); + } + return inReadLock( kvLock, () -> { @@ -1267,6 +1521,10 @@ public List prefixLookup(byte[] prefixKey) throws IOException { } public List limitScan(int limit) throws IOException { + if (lifecycle != null) { + return requireOpenedTablet().limitScan(limit); + } + return inReadLock( kvLock, () -> { @@ -1294,6 +1552,10 @@ private List toValueBodySlices(List values) { */ public OpenScanResult openScan(String scannerId, long limit, long initialAccessTimeMs) throws IOException { + if (lifecycle != null) { + return requireOpenedTablet().openScan(scannerId, limit, initialAccessTimeMs); + } + return inReadLock( kvLock, () -> { @@ -1353,6 +1615,10 @@ public OpenScanResult openScan(String scannerId, long limit, long initialAccessT } public KvBatchWriter createKvBatchWriter() { + if (lifecycle != null) { + return requireOpenedTablet().createKvBatchWriter(); + } + return rocksDBKv.newWriteBatch( writeBatchSize, serverMetricGroup.kvFlushCount(), @@ -1371,6 +1637,10 @@ public void close() throws Exception { } public void close(KvCloseMode closeMode) throws Exception { + if (lifecycle != null) { + lifecycle.closeKvLazy(closeMode); + return; + } LOG.info( "Close kv tablet {} for table {} with mode {}.", tableBucket, @@ -1401,6 +1671,11 @@ public void close(KvCloseMode closeMode) throws Exception { /** Completely delete the kv directory and all contents form the file system with no delay. */ public void drop() throws Exception { + if (lifecycle != null) { + lifecycle.dropKvLazy(); + return; + } + inWriteLock( kvLock, () -> { @@ -1417,6 +1692,15 @@ public RocksIncrementalSnapshot createIncrementalSnapshot( KvSnapshotDataUploader kvSnapshotDataUploader, long lastCompletedSnapshotId, Counter remoteKvCopyBytes) { + if (lifecycle != null) { + return requireOpenedTablet() + .createIncrementalSnapshot( + uploadedSstFiles, + kvSnapshotDataUploader, + lastCompletedSnapshotId, + remoteKvCopyBytes); + } + return new RocksIncrementalSnapshot( uploadedSstFiles, rocksDBKv.getDb(), @@ -1430,17 +1714,32 @@ public RocksIncrementalSnapshot createIncrementalSnapshot( // only for testing. @VisibleForTesting KvPreWriteBuffer getKvPreWriteBuffer() { + if (lifecycle != null) { + KvTablet opened = openedTablet; + return opened == null ? null : opened.getKvPreWriteBuffer(); + } + return kvPreWriteBuffer; } // only for testing. @VisibleForTesting public RocksDBKv getRocksDBKv() { + if (lifecycle != null) { + KvTablet opened = openedTablet; + return opened == null ? null : opened.getRocksDBKv(); + } + return rocksDBKv; } /** Returns the recent normalized backpressure pressure in {@code [0, 1)}. */ public float currentPressure() { + if (lifecycle != null) { + KvTablet opened = openedTablet; + return opened == null ? 0F : opened.currentPressure(); + } + return rocksDBKv.currentPressure(); } @@ -1520,4 +1819,111 @@ enum FlushState { RUNNING, STORAGE_BLOCKED } + + /** Returns true if this tablet is in lazy mode (as opposed to eager/traditional mode). */ + public boolean isLazyMode() { + return lifecycle != null; + } + + /** Returns true if this tablet is in lazy mode and currently OPEN (RocksDB loaded). */ + public boolean isLazyOpen() { + return lifecycle != null && lifecycle.isOpen(); + } + + /** Returns the current lazy state. */ + public KvTabletLazyLifecycle.LazyState getLazyState() { + return lifecycle != null ? lifecycle.getLazyState() : null; + } + + /** + * Acquire a guard that prevents RocksDB from being released while held. In eager mode, returns + * a no-op guard immediately. In lazy mode, ensures RocksDB is open (blocking if necessary) and + * pins it. + * + *

Must be called OUTSIDE any Replica-level locks (e.g. leaderIsrUpdateLock) to avoid + * blocking leader transitions during slow opens. + */ + public Guard acquireGuard() { + return lifecycle != null ? lifecycle.acquireGuard() : Guard.NOOP; + } + + /** + * Try to acquire a guard only if RocksDB is already open. + * + *

Returns {@code null} for a lazy tablet that is currently LAZY/FAILED/RELEASING/CLOSED. + */ + @Nullable + Guard tryAcquireExistingGuard() { + return lifecycle != null ? lifecycle.tryAcquireExistingGuard() : Guard.NOOP; + } + + /** Release a pin. Called by Guard.close(). */ + void releasePin() { + if (lifecycle != null) { + lifecycle.releasePin(); + } + } + + /** Pre-check for idle release eligibility. */ + public boolean canRelease(long closeIdleIntervalMs, long nowMs) { + return lifecycle != null && lifecycle.canRelease(closeIdleIntervalMs, nowMs); + } + + public boolean releaseKv() { + return lifecycle != null && lifecycle.releaseKv(); + } + + public void dropKvLazy() { + if (lifecycle != null) { + lifecycle.dropKvLazy(); + } + } + + // ---- Lazy state queries ---- + + public long getCachedRowCount() { + return lifecycle != null ? lifecycle.getCachedRowCount() : 0; + } + + public long getCachedFlushedLogOffset() { + return lifecycle != null ? lifecycle.getCachedFlushedLogOffset() : 0; + } + + public long getLastAccessTimestamp() { + return lifecycle != null ? lifecycle.getLastAccessTimestamp() : 0; + } + + /** + * Returns the number of active pins preventing RocksDB release. + * + *

This is primarily for testing and monitoring purposes. Exposing internal concurrency state + * may lead to misuse in production code. + */ + @VisibleForTesting + public int getActivePins() { + return lifecycle != null ? lifecycle.getActivePins() : 0; + } + + // ---- Test helpers ---- + + @VisibleForTesting + void setLazyStateForTesting(KvTabletLazyLifecycle.LazyState state) { + if (lifecycle != null) { + lifecycle.setLazyStateForTesting(state); + } + } + + @VisibleForTesting + void setLastAccessTimestampForTesting(long timestamp) { + if (lifecycle != null) { + lifecycle.setLastAccessTimestampForTesting(timestamp); + } + } + + @VisibleForTesting + void setClockForTesting(Clock clock) { + if (lifecycle != null) { + lifecycle.setClockForTesting(clock); + } + } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTabletLazyLifecycle.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTabletLazyLifecycle.java new file mode 100644 index 00000000000..07dc8f82e9d --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTabletLazyLifecycle.java @@ -0,0 +1,1056 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.kv; + +import org.apache.fluss.annotation.VisibleForTesting; +import org.apache.fluss.exception.KvStorageException; +import org.apache.fluss.server.metrics.group.TabletServerMetricGroup; +import org.apache.fluss.utils.ExponentialBackoff; +import org.apache.fluss.utils.FileUtils; +import org.apache.fluss.utils.clock.Clock; +import org.apache.fluss.utils.clock.SystemClock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.GuardedBy; + +import java.io.File; +import java.util.concurrent.CancellationException; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.IntSupplier; +import java.util.function.Supplier; + +/** + * Manages the lazy lifecycle state machine for a {@link KvTablet}. Encapsulates all state + * transitions (LAZY → OPENING → OPEN → RELEASING → LAZY), pin management, open/release/drop logic, + * and callback orchestration. + * + *

This class is only instantiated for lazy-mode tablets. EAGER-mode tablets do not use this + * class. + */ +public final class KvTabletLazyLifecycle { + private static final Logger LOG = LoggerFactory.getLogger(KvTabletLazyLifecycle.class); + + /** Coarsen access timestamp updates to reduce volatile writes on hot path. */ + private static final long ACCESS_TIMESTAMP_GRANULARITY_MS = 1000; + + // ---- Lazy lifecycle states ---- + + /** Internal RocksDB lazy lifecycle states. */ + enum LazyState { + /** Lazy mode: RocksDB not yet opened. */ + LAZY, + /** Lazy mode: RocksDB is being opened by another thread. */ + OPENING, + /** Lazy mode: RocksDB is open and serving requests. */ + OPEN, + /** Lazy mode: RocksDB is being released (closing without deleting data). */ + RELEASING, + /** Lazy mode: last open attempt failed, in backoff cooldown. */ + FAILED, + /** + * Terminal state: tablet is closed and will not be reopened. Local data may or may not have + * been deleted depending on whether close or drop was called. + */ + CLOSED + } + + // ---- Callback interfaces ---- + + /** + * Callback for the actual RocksDB open work. Provided by Replica since it requires access to + * snapshot context, log recovery, etc. + */ + @FunctionalInterface + public interface OpenCallback { + /** Performs the open and returns the opened RocksDB KvTablet instance. */ + KvTablet doOpen(boolean hasLocalData) throws Exception; + } + + /** + * Callback invoked after a successful open commit. Used by Replica to install the tablet + * reference and start periodic snapshot. + */ + @FunctionalInterface + public interface OpenCommitCallback { + /** Called after RocksDB state has been installed into the sentinel. */ + void onOpenCommitted(KvTablet kvTablet); + } + + /** Callback for releasing (closing) RocksDB without destroying local data. */ + @FunctionalInterface + public interface ReleaseCallback { + /** Performs release-time cleanup (e.g. stop snapshots, unregister metrics). */ + void doRelease(KvTablet kvTablet); + } + + /** Callback for dropping (destroying) a KvTablet and cleaning up associated resources. */ + @FunctionalInterface + public interface DropCallback { + /** Performs drop-time cleanup (e.g. unregister metrics, stop snapshots). */ + void doDrop(KvTablet kvTablet); + } + + // ---- The owning tablet ---- + + private final KvTablet tablet; + + // ---- State machine fields ---- + + private volatile LazyState lazyState = LazyState.LAZY; + + /** Lock guarding lazy state transitions. */ + private final ReentrantLock lazyStateLock = new ReentrantLock(); + + private final Condition lazyStateChanged = lazyStateLock.newCondition(); + + /** Active pin count — prevents release while operations are in-flight. */ + private final AtomicInteger activePins = new AtomicInteger(0); + + private volatile boolean rejectNewPins; + + /** Generation counter for fencing stale open results. */ + private long openGeneration; + + /** Semaphore for throttling concurrent open operations across all tablets. */ + private @Nullable Semaphore openSemaphore; + + private long openTimeoutMs; + private long releaseDrainTimeoutMs; + private @Nullable ExponentialBackoff failedBackoff; + private Clock clock = SystemClock.getInstance(); + + /** FAILED state tracking. */ + private long failedTimestamp; + + private int failureCount; + private @Nullable Throwable lastFailureCause; + + /** Cached values served when RocksDB is not open. */ + private volatile long cachedRowCount; + + private volatile long cachedFlushedLogOffset; + + /** Whether local RocksDB data directory exists from a previous open. */ + @GuardedBy("lazyStateLock") + private boolean hasLocalData; + + /** Access timestamp for idle release decisions (coarsened to reduce volatile writes). */ + private volatile long lastAccessTimestamp; + + /** Epoch suppliers for fencing stale open results. */ + private @Nullable IntSupplier leaderEpochSupplier; + + private @Nullable IntSupplier bucketEpochSupplier; + + /** Supplier for tablet directory path — used by sentinel in LAZY/FAILED state. */ + private @Nullable Supplier tabletDirSupplier; + + /** Callbacks (set by Replica after construction). */ + private @Nullable OpenCallback openCallback; + + private @Nullable OpenCommitCallback commitCallback; + private @Nullable DropCallback dropCallback; + private @Nullable ReleaseCallback releaseCallback; + + /** Tracks post-open finalization that runs after OPEN becomes externally visible. */ + private boolean commitCallbackInFlight; + + KvTabletLazyLifecycle(KvTablet tablet) { + this.tablet = tablet; + } + + // ---- Configuration (called by Replica after sentinel creation) ---- + + /** + * Configure lazy open parameters. Must be called before any {@link #acquireGuard()} calls. + * + * @param clock the clock for time-based operations + * @param openSemaphore semaphore to limit concurrent open operations (can be null) + * @param openTimeoutMs timeout for open operations in milliseconds + * @param failedBackoffBaseMs base backoff time for failed opens in milliseconds + * @param failedBackoffMaxMs max backoff time for failed opens in milliseconds + * @param releaseDrainTimeoutMs timeout for draining pins during release in milliseconds + */ + public void configureLazyOpen( + Clock clock, + Semaphore openSemaphore, + long openTimeoutMs, + long failedBackoffBaseMs, + long failedBackoffMaxMs, + long releaseDrainTimeoutMs) { + this.clock = clock; + this.openSemaphore = openSemaphore; + this.openTimeoutMs = openTimeoutMs; + this.failedBackoff = new ExponentialBackoff(failedBackoffBaseMs, 2, failedBackoffMaxMs, 0); + this.releaseDrainTimeoutMs = releaseDrainTimeoutMs; + // Register initial LAZY state in metrics + updateStateGauges(null, LazyState.LAZY); + } + + /** + * Sets the callback responsible for performing the actual RocksDB open operation. This is + * provided by Replica since it requires access to snapshot context, log recovery, etc. + * + * @param callback the open callback to invoke when transitioning from LAZY to OPEN + */ + public void setOpenCallback(OpenCallback callback) { + this.openCallback = callback; + } + + /** + * Sets the callback invoked after a successful open commit. Used by Replica to install the + * tablet reference and start periodic snapshot. + * + * @param callback the commit callback to invoke after RocksDB state is installed + */ + public void setCommitCallback(OpenCommitCallback callback) { + this.commitCallback = callback; + } + + /** + * Sets the callback for full-state cleanup when the tablet is dropped. Handles all states, + * waits for in-progress operations, then transitions to CLOSED. + * + * @param callback the drop callback to invoke during tablet deletion + */ + public void setDropCallback(DropCallback callback) { + this.dropCallback = callback; + } + + /** + * Sets the callback for releasing (closing) RocksDB without destroying local data. Called by + * the idle release controller when transitioning from OPEN to LAZY. + * + * @param callback the release callback to invoke during idle release + */ + public void setReleaseCallback(ReleaseCallback callback) { + this.releaseCallback = callback; + } + + /** + * Sets the supplier for the tablet directory path. Used by sentinel in LAZY/FAILED state for + * cleanup operations. + * + * @param supplier the directory supplier + */ + public void setTabletDirSupplier(Supplier supplier) { + this.tabletDirSupplier = supplier; + } + + /** Returns the tablet directory from the supplier, or null if not configured. */ + @Nullable + File getTabletDir() { + return tabletDirSupplier != null ? tabletDirSupplier.get() : null; + } + + /** + * Sets the supplier for the leader epoch used in fencing stale open operations. + * + * @param supplier the leader epoch supplier + */ + public void setLeaderEpochSupplier(IntSupplier supplier) { + this.leaderEpochSupplier = supplier; + } + + /** + * Sets the supplier for the bucket epoch used in fencing stale open operations. + * + * @param supplier the bucket epoch supplier + */ + public void setBucketEpochSupplier(IntSupplier supplier) { + this.bucketEpochSupplier = supplier; + } + + /** + * Initializes the cached row count from snapshot metadata. Called during sentinel creation to + * serve read queries while in LAZY state before the first open. Also sets the sentinel's + * rowCount field so that OPENING state reads return the correct value. + * + * @param rowCount the initial row count (-1 if unknown/disabled) + */ + public void initCachedRowCount(long rowCount) { + this.cachedRowCount = rowCount; + tablet.setRowCount(rowCount); + } + + // ---- State queries ---- + + /** Returns whether the tablet is currently in OPEN state with RocksDB loaded and serving. */ + boolean isOpen() { + return lazyState == LazyState.OPEN; + } + + /** Returns the current lifecycle state of this lazy tablet. */ + LazyState getLazyState() { + return lazyState; + } + + /** + * Returns the cached row count from the last flush or release operation. This value may be + * stale but is sufficient for serving read queries while in LAZY state. + */ + long getCachedRowCount() { + return cachedRowCount; + } + + /** + * Returns the flushed log offset cached from the last flush or release operation. Used to + * determine how much incremental log needs to be replayed when reopening from local data. + */ + long getCachedFlushedLogOffset() { + return cachedFlushedLogOffset; + } + + /** + * Returns the timestamp (in milliseconds) of the last access to this tablet. Used by the idle + * release controller to determine eligibility for release. + */ + long getLastAccessTimestamp() { + return lastAccessTimestamp; + } + + /** + * Returns the number of active pins that are currently preventing RocksDB from being released. + * Each pin corresponds to a {@link KvTablet.Guard} held by an in-flight operation. + */ + int getActivePins() { + return activePins.get(); + } + + /** + * Returns true if this tablet is in LAZY or FAILED state (RocksDB not open), meaning + * KvTablet.getRowCount() should return the cached value instead of querying RocksDB. + */ + boolean needsCachedRowCount() { + LazyState s = lazyState; + return s == LazyState.LAZY || s == LazyState.FAILED; + } + + // ---- Guard / Pin management ---- + + /** + * Acquire a guard that prevents RocksDB from being released while held. Ensures RocksDB is open + * (blocking if necessary) and pins it. + * + *

Must be called OUTSIDE any Replica-level locks (e.g. leaderIsrUpdateLock) to avoid + * blocking leader transitions during slow opens. + * + * @throws IllegalStateException if the lifecycle has not been fully configured via {@link + * #configureLazyOpen} and callback setters + */ + KvTablet.Guard acquireGuard() { + // Defensive check: ensure all required callbacks are set before allowing access + if (openCallback == null) { + throw new IllegalStateException( + "KvTabletLazyLifecycle not fully configured: openCallback is null. " + + "Ensure configureLazyOpen() and setOpenCallback() are called before use."); + } + + // Fast path: try to pin without blocking + KvTablet.Guard fast = tryAcquirePinAsGuard(); + if (fast != null) { + return fast; + } + + // Slow path: ensure open, then pin + ensureOpen(); + // Between ensureOpen() returning and pin, a concurrent release could happen. + // Retry up to 3 times. + for (int attempt = 0; ; attempt++) { + lazyStateLock.lock(); + try { + if (lazyState == LazyState.OPEN && !rejectNewPins) { + activePins.incrementAndGet(); + touchAccessTimestamp(); + return new KvTablet.Guard(tablet); + } + if (lazyState == LazyState.CLOSED) { + throw new KvStorageException( + "KvTablet is closed for " + tablet.getTableBucket()); + } + } finally { + lazyStateLock.unlock(); + } + if (attempt >= 2) { + throw new KvStorageException( + "Failed to pin KvTablet after open for " + tablet.getTableBucket()); + } + // Re-open and retry + ensureOpen(); + } + } + + /** + * Try to acquire a guard only if RocksDB is already OPEN. + * + *

Unlike {@link #acquireGuard()}, this method never triggers a lazy open. It is intended for + * background maintenance paths (flush, snapshot init) that must coordinate with idle release + * but should not reopen a lazy tablet on their own. + */ + @Nullable + KvTablet.Guard tryAcquireExistingGuard() { + return tryAcquirePinAsGuard(); + } + + /** + * Fast-path pin attempt. Returns Guard if OPEN and accepting pins; null otherwise. + * + *

Safety relies on the double-check pattern: we optimistically increment the pin count, then + * re-verify the state. If the state changed concurrently (e.g., release started), the re-check + * fails and we undo the increment. This is safe on all architectures (x86, ARM) regardless of + * volatile read ordering between independent variables. + */ + @Nullable + private KvTablet.Guard tryAcquirePinAsGuard() { + if (lazyState == LazyState.OPEN && !rejectNewPins) { + activePins.incrementAndGet(); + if (lazyState == LazyState.OPEN && !rejectNewPins) { + touchAccessTimestamp(); + return new KvTablet.Guard(tablet); + } + decrementAndMaybeSignal(); + } + return null; + } + + /** Release a pin. Called by Guard.close(). */ + void releasePin() { + decrementAndMaybeSignal(); + } + + private void decrementAndMaybeSignal() { + if (activePins.decrementAndGet() == 0) { + if (rejectNewPins) { + lazyStateLock.lock(); + try { + lazyStateChanged.signalAll(); + } finally { + lazyStateLock.unlock(); + } + } + } + } + + private void touchAccessTimestamp() { + long now = clock.milliseconds(); + if (now - lastAccessTimestamp > ACCESS_TIMESTAMP_GRANULARITY_MS) { + lastAccessTimestamp = now; + } + } + + /** + * Update state gauge metrics on state transitions. Only LAZY, OPEN, FAILED are tracked as + * "stable" states; OPENING, RELEASING, CLOSED fall into default (no-op). + */ + private void updateStateGauges(@Nullable LazyState from, LazyState to) { + TabletServerMetricGroup metricGroup = tablet.serverMetricGroup; + if (metricGroup == null) { + return; + } + if (from != null) { + switch (from) { + case LAZY: + metricGroup.kvTabletLazyCount().decrementAndGet(); + break; + case OPEN: + metricGroup.kvTabletOpenCount().decrementAndGet(); + break; + case FAILED: + metricGroup.kvTabletFailedCount().decrementAndGet(); + break; + default: + break; + } + } + switch (to) { + case LAZY: + metricGroup.kvTabletLazyCount().incrementAndGet(); + break; + case OPEN: + metricGroup.kvTabletOpenCount().incrementAndGet(); + break; + case FAILED: + metricGroup.kvTabletFailedCount().incrementAndGet(); + break; + default: + break; + } + } + + // ---- Open management ---- + + /** + * Ensures RocksDB is open. If already OPEN, returns immediately. If LAZY or FAILED (past + * cooldown), triggers a slow-path open. If OPENING or RELEASING, blocks until state changes. + */ + private void ensureOpen() { + lazyStateLock.lock(); + try { + while (true) { + switch (lazyState) { + case OPEN: + return; + + case OPENING: + if (!lazyStateChanged.await(openTimeoutMs, TimeUnit.MILLISECONDS)) { + throw new KvStorageException( + "KvTablet open timed out for " + tablet.getTableBucket()); + } + continue; + + case RELEASING: + if (!lazyStateChanged.await(openTimeoutMs, TimeUnit.MILLISECONDS)) { + throw new KvStorageException( + "KvTablet open timed out waiting for release: " + + tablet.getTableBucket()); + } + continue; + + case FAILED: + long elapsed = clock.milliseconds() - failedTimestamp; + long cooldown = failedBackoff.backoff(failureCount); + if (elapsed < cooldown) { + throw new KvStorageException( + "KvTablet open failed for " + + tablet.getTableBucket() + + ", remaining cooldown: " + + (cooldown - elapsed) + + " ms", + lastFailureCause); + } + // Fall through to LAZY — cooldown expired, retry open. + + case LAZY: + updateStateGauges(lazyState, LazyState.OPENING); + lazyState = LazyState.OPENING; + openGeneration++; + long myGeneration = openGeneration; + int myLeaderEpoch = + leaderEpochSupplier != null ? leaderEpochSupplier.getAsInt() : -1; + int myBucketEpoch = + bucketEpochSupplier != null ? bucketEpochSupplier.getAsInt() : -1; + boolean localDataExists = hasLocalData; + lazyStateLock.unlock(); + try { + doSlowOpen(myGeneration, myLeaderEpoch, myBucketEpoch, localDataExists); + } catch (Throwable t) { + // doSlowOpen already called commitOpenResult() which transitions + // to FAILED, so no need to handle state transition here. + if (t instanceof RuntimeException) { + throw (RuntimeException) t; + } + throw new KvStorageException( + "KvTablet open failed for " + tablet.getTableBucket(), t); + } + return; + + case CLOSED: + throw new KvStorageException( + "KvTablet is closed for " + tablet.getTableBucket()); + + default: + throw new IllegalStateException("Unexpected state: " + lazyState); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new KvStorageException( + "Interrupted waiting for KvTablet open: " + tablet.getTableBucket(), e); + } finally { + if (lazyStateLock.isHeldByCurrentThread()) { + lazyStateLock.unlock(); + } + } + } + + private void doSlowOpen( + long myGeneration, int myLeaderEpoch, int myBucketEpoch, boolean localDataExists) + throws Exception { + KvTablet newTablet = null; + boolean committed = false; + boolean semaphoreAcquired = false; + + try { + if (openSemaphore != null) { + semaphoreAcquired = openSemaphore.tryAcquire(openTimeoutMs, TimeUnit.MILLISECONDS); + if (!semaphoreAcquired) { + throw new KvStorageException( + "KvTablet open semaphore timeout for " + tablet.getTableBucket()); + } + } + + newTablet = openCallback.doOpen(localDataExists); + committed = + commitOpenResult(myGeneration, myLeaderEpoch, myBucketEpoch, newTablet, null); + if (!committed) { + throw new CancellationException( + "open result fenced by concurrent epoch/generation change"); + } + } catch (Exception e) { + commitOpenResult(myGeneration, myLeaderEpoch, myBucketEpoch, null, e); + throw e; + } finally { + if (!committed && newTablet != null) { + try { + newTablet.close(); + } catch (Exception ignored) { + } + // Clean up the directory created by the failed open attempt BEFORE + // releasing the semaphore, so that a subsequent open cannot start + // writing to the same canonical path while deletion is in progress. + File failedDir = newTablet.getKvTabletDir(); + if (failedDir != null) { + FileUtils.deleteDirectoryQuietly(failedDir); + } + } + if (semaphoreAcquired) { + openSemaphore.release(); + } + } + } + + private boolean commitOpenResult( + long myGeneration, + int myLeaderEpoch, + int myBucketEpoch, + @Nullable KvTablet newTablet, + @Nullable Exception error) { + lazyStateLock.lock(); + try { + if (openGeneration != myGeneration) { + if (lazyState == LazyState.OPENING) { + transitionToFailed( + new CancellationException( + "open superseded by generation " + openGeneration), + true); + } + return false; + } + + if (error != null) { + transitionToFailed(error, false); + return false; + } + + // Epoch fencing + int currentLeaderEpoch = + leaderEpochSupplier != null ? leaderEpochSupplier.getAsInt() : -1; + int currentBucketEpoch = + bucketEpochSupplier != null ? bucketEpochSupplier.getAsInt() : -1; + if (currentLeaderEpoch != myLeaderEpoch || currentBucketEpoch != myBucketEpoch) { + transitionToFailed(new CancellationException("epoch changed during open"), true); + return false; + } + + // Success — absorb RocksDB state from the newly opened tablet. + // Note: we set OPEN and signal waiters before running commitCallback (which starts + // the snapshot manager). Writes during this brief window are safe — data is in + // RocksDB and the WAL. The commitCallbackInFlight flag prevents release until the + // callback completes. + tablet.installRocksDB(newTablet); + lazyState = LazyState.OPEN; + failureCount = 0; + lastAccessTimestamp = clock.milliseconds(); + rejectNewPins = false; + commitCallbackInFlight = true; + lazyStateChanged.signalAll(); + } finally { + lazyStateLock.unlock(); + } + + try { + updateStateGauges(LazyState.OPENING, LazyState.OPEN); + + // Invoke commit callback outside the lock to avoid blocking acquireGuard() callers. + if (commitCallback != null) { + commitCallback.onOpenCommitted(tablet); + } + } catch (Exception e) { + LOG.warn( + "Post-open commit callback failed for {}, keeping tablet OPEN", + tablet.getTableBucket(), + e); + } finally { + lazyStateLock.lock(); + try { + commitCallbackInFlight = false; + lazyStateChanged.signalAll(); + } finally { + lazyStateLock.unlock(); + } + } + return true; + } + + /** Must be called while holding {@code lazyStateLock}. */ + private void transitionToFailed(Throwable cause, boolean resetCount) { + updateStateGauges(LazyState.OPENING, LazyState.FAILED); + lazyState = LazyState.FAILED; + failedTimestamp = clock.milliseconds(); + if (resetCount) { + failureCount = 0; + } else { + failureCount++; + } + lastFailureCause = cause; + lazyStateChanged.signalAll(); + } + + // ---- Release logic (idle release by KvManager) ---- + + /** Pre-check for idle release eligibility. */ + boolean canRelease(long closeIdleIntervalMs, long nowMs) { + if (lazyState != LazyState.OPEN) { + return false; + } + if (activePins.get() > 0) { + return false; + } + return nowMs - lastAccessTimestamp >= closeIdleIntervalMs; + } + + /** + * Release RocksDB resources: OPEN -> RELEASING -> LAZY. Caches metadata before close for + * serving queries while in LAZY state. + * + * @return true if release succeeded, false if aborted + */ + boolean releaseKv() { + // Pre-check state outside lock as a fast-path rejection + if (lazyState != LazyState.OPEN) { + return false; + } + + lazyStateLock.lock(); + try { + if (lazyState != LazyState.OPEN) { + return false; + } + + // Transition to RELEASING — write rejectNewPins before lazyState so that + // tryAcquirePinAsGuard() sees the reject flag if it reads OPEN. + rejectNewPins = true; + lazyState = LazyState.RELEASING; + + if (!drainPins()) { + LOG.warn("{} release drain timeout, aborting release", tablet.getTableBucket()); + transitionBackToOpen(); + return false; + } + + // Re-check flush progress only after blocking new pins and draining + // all in-flight operations. This closes the race where a writer acquired + // a guard before RELEASING but appended after an earlier LEO check. + long logEndOffset = tablet.logTablet.localLogEndOffset(); + if (tablet.getFlushedLogOffset() < logEndOffset || tablet.hasActiveResourceLeases()) { + transitionBackToOpen(); + return false; + } + + if (!awaitCommitCallbackCompletion()) { + LOG.warn( + "{} release waited too long for open commit finalization, aborting release", + tablet.getTableBucket()); + transitionBackToOpen(); + return false; + } + + // Cache metadata AFTER drain — all pinned operations have completed, + // so rowCount reflects the latest flushed state. + cachedRowCount = tablet.currentRowCount(); + cachedFlushedLogOffset = tablet.getFlushedLogOffset(); + } finally { + lazyStateLock.unlock(); + } + + // All pins drained, safe to close + boolean releaseSucceeded = false; + try { + if (releaseCallback != null) { + releaseCallback.doRelease(tablet); + } + tablet.detachRocksDB(); + releaseSucceeded = true; + } catch (Exception e) { + LOG.warn( + "{} release callback failed, rolling back to OPEN", tablet.getTableBucket(), e); + } finally { + lazyStateLock.lock(); + try { + if (releaseSucceeded) { + lazyState = LazyState.LAZY; + hasLocalData = true; + updateStateGauges(LazyState.OPEN, LazyState.LAZY); + rejectNewPins = false; + lazyStateChanged.signalAll(); + } else { + transitionBackToOpen(); + } + } finally { + lazyStateLock.unlock(); + } + } + return releaseSucceeded; + } + + // ---- Drop logic ---- + + /** + * Full-state cleanup for lazy mode: handles all states, waits for in-progress operations, then + * transitions to CLOSED. Called by Replica.dropKv(). + */ + void dropKvLazy() { + lazyStateLock.lock(); + try { + switch (lazyState) { + case CLOSED: + break; + + case LAZY: + case FAILED: + tablet.deleteLocalDirectory(); + break; + + case OPENING: + // Fence the in-progress open so commitOpenResult() will reject it. + // If awaitStateExit times out (open thread stuck on semaphore or slow I/O), + // the open thread will eventually call commitOpenResult(), find the generation + // mismatch, and transition to FAILED. The final CLOSED transition at the + // bottom of this method will override FAILED. The open thread's finally block + // in doSlowOpen will close the newTablet, preventing resource leaks. + openGeneration++; + awaitStateExit(LazyState.OPENING, openTimeoutMs * 2); + if (lazyState == LazyState.OPEN) { + rejectNewPins = true; + awaitCommitCallbackCompletion(); + drainPinsForDrop(); + dropOpenTablet(); + } else if (lazyState != LazyState.OPENING) { + tablet.deleteLocalDirectory(); + } + break; + + case OPEN: + rejectNewPins = true; + awaitCommitCallbackCompletion(); + drainPinsForDrop(); + dropOpenTablet(); + break; + + case RELEASING: + awaitStateExit(LazyState.RELEASING, releaseDrainTimeoutMs * 2); + if (lazyState == LazyState.OPEN) { + rejectNewPins = true; + awaitCommitCallbackCompletion(); + drainPinsForDrop(); + dropOpenTablet(); + } else if (lazyState != LazyState.RELEASING) { + tablet.deleteLocalDirectory(); + } + break; + + default: + throw new IllegalStateException("Unexpected state in dropKvLazy: " + lazyState); + } + + LazyState stateBeforeClose = lazyState; + if (stateBeforeClose != LazyState.CLOSED) { + updateStateGauges(stateBeforeClose, LazyState.CLOSED); + } + lazyState = LazyState.CLOSED; + hasLocalData = false; + rejectNewPins = false; + failureCount = 0; + failedTimestamp = 0; + lastFailureCause = null; + lazyStateChanged.signalAll(); + } finally { + lazyStateLock.unlock(); + } + } + + private void drainPinsForDrop() { + if (!drainPins()) { + LOG.warn( + "{} drop drain timeout with {} active pins, proceeding with close", + tablet.getTableBucket(), + activePins.get()); + } + } + + private boolean drainPins() { + long deadline = clock.milliseconds() + releaseDrainTimeoutMs; + while (activePins.get() > 0) { + long remaining = deadline - clock.milliseconds(); + if (remaining <= 0) { + return false; + } + try { + lazyStateChanged.await(remaining, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + return true; + } + + /** Waits under {@code lazyStateLock} for post-open finalization to complete. */ + private boolean awaitCommitCallbackCompletion() { + long deadline = clock.milliseconds() + releaseDrainTimeoutMs; + while (commitCallbackInFlight) { + long remaining = deadline - clock.milliseconds(); + if (remaining <= 0) { + return false; + } + try { + if (!lazyStateChanged.await(remaining, TimeUnit.MILLISECONDS)) { + return false; + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + return true; + } + + /** + * Wait (under lazyStateLock) until {@code lazyState} is no longer {@code state}, or timeout. + */ + private void awaitStateExit(LazyState state, long timeoutMs) { + long deadlineNanos = clock.nanoseconds() + TimeUnit.MILLISECONDS.toNanos(timeoutMs); + while (lazyState == state) { + try { + long remainNanos = deadlineNanos - clock.nanoseconds(); + if (remainNanos <= 0 + || !lazyStateChanged.await(remainNanos, TimeUnit.NANOSECONDS)) { + break; + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + } + + private void doDropViaCallback() { + if (dropCallback != null) { + dropCallback.doDrop(tablet); + } + } + + void closeKvLazy(KvCloseMode closeMode) { + lazyStateLock.lock(); + try { + switch (lazyState) { + case CLOSED: + return; + case LAZY: + case FAILED: + break; + case OPENING: + openGeneration++; + awaitStateExit(LazyState.OPENING, openTimeoutMs * 2); + if (lazyState == LazyState.OPEN) { + rejectNewPins = true; + awaitCommitCallbackCompletion(); + drainPins(); + tablet.detachRocksDB(closeMode); + } + break; + case OPEN: + rejectNewPins = true; + awaitCommitCallbackCompletion(); + drainPins(); + tablet.detachRocksDB(closeMode); + break; + case RELEASING: + awaitStateExit(LazyState.RELEASING, releaseDrainTimeoutMs * 2); + if (lazyState == LazyState.OPEN) { + rejectNewPins = true; + awaitCommitCallbackCompletion(); + drainPins(); + tablet.detachRocksDB(closeMode); + } + break; + default: + throw new IllegalStateException( + "Unexpected state in closeKvLazy: " + lazyState); + } + + LazyState stateBeforeClose = lazyState; + if (stateBeforeClose != LazyState.CLOSED) { + updateStateGauges(stateBeforeClose, LazyState.CLOSED); + } + lazyState = LazyState.CLOSED; + hasLocalData = false; + rejectNewPins = false; + failureCount = 0; + failedTimestamp = 0; + lastFailureCause = null; + lazyStateChanged.signalAll(); + } finally { + lazyStateLock.unlock(); + } + } + + /** Must be called while holding {@code lazyStateLock} and after pins have drained. */ + private void dropOpenTablet() { + doDropViaCallback(); + tablet.detachRocksDB(KvCloseMode.DISCARD_UNPERSISTED_STATE); + tablet.deleteLocalDirectory(); + } + + /** Must be called while holding {@code lazyStateLock}. */ + private void transitionBackToOpen() { + lazyState = LazyState.OPEN; + rejectNewPins = false; + lazyStateChanged.signalAll(); + } + + // ---- Test helpers ---- + + @VisibleForTesting + void setLazyStateForTesting(LazyState state) { + lazyStateLock.lock(); + try { + this.lazyState = state; + lazyStateChanged.signalAll(); + } finally { + lazyStateLock.unlock(); + } + } + + @VisibleForTesting + void setLastAccessTimestampForTesting(long timestamp) { + this.lastAccessTimestamp = timestamp; + } + + @VisibleForTesting + void setClockForTesting(Clock clock) { + this.clock = clock; + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TabletServerMetricGroup.java b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TabletServerMetricGroup.java index dac5562c10d..ad50eac3146 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TabletServerMetricGroup.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/metrics/group/TabletServerMetricGroup.java @@ -34,6 +34,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.LongSupplier; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -72,6 +73,9 @@ public class TabletServerMetricGroup extends AbstractMetricGroup { private final Histogram kvFlushLatencyHistogram; private final Counter kvTruncateAsDuplicatedCount; private final Counter kvTruncateAsErrorCount; + private final AtomicInteger kvTabletOpenCount = new AtomicInteger(0); + private final AtomicInteger kvTabletLazyCount = new AtomicInteger(0); + private final AtomicInteger kvTabletFailedCount = new AtomicInteger(0); // aggregated replica metrics private final Counter isrShrinks; @@ -140,6 +144,9 @@ public TabletServerMetricGroup( meter( MetricNames.KV_PRE_WRITE_BUFFER_TRUNCATE_AS_ERROR_RATE, new MeterView(kvTruncateAsErrorCount)); + gauge(MetricNames.KV_TABLET_OPEN_COUNT, kvTabletOpenCount::get); + gauge(MetricNames.KV_TABLET_LAZY_COUNT, kvTabletLazyCount::get); + gauge(MetricNames.KV_TABLET_FAILED_COUNT, kvTabletFailedCount::get); // replica metrics isrExpands = new SimpleCounter(); @@ -301,6 +308,21 @@ public Counter kvTruncateAsErrorCount() { return kvTruncateAsErrorCount; } + /** Returns the gauge-backing counter for currently open lazy KvTablets. */ + public AtomicInteger kvTabletOpenCount() { + return kvTabletOpenCount; + } + + /** Returns the gauge-backing counter for KvTablets in LAZY (released) state. */ + public AtomicInteger kvTabletLazyCount() { + return kvTabletLazyCount; + } + + /** Returns the gauge-backing counter for KvTablets in FAILED state. */ + public AtomicInteger kvTabletFailedCount() { + return kvTabletFailedCount; + } + public Counter isrShrinks() { return isrShrinks; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index e3eda80a82c..4ff9f24e2c3 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -63,6 +63,7 @@ import org.apache.fluss.server.kv.KvRecoverHelper; import org.apache.fluss.server.kv.KvStateLookupResult; import org.apache.fluss.server.kv.KvTablet; +import org.apache.fluss.server.kv.KvTabletLazyLifecycle; import org.apache.fluss.server.kv.RemoteLogFetcher; import org.apache.fluss.server.kv.autoinc.AutoIncIDRange; import org.apache.fluss.server.kv.historical.HistoricalValueLookup; @@ -112,6 +113,7 @@ import org.apache.fluss.types.RowType; import org.apache.fluss.utils.ByteArraySlice; import org.apache.fluss.utils.CloseableRegistry; +import org.apache.fluss.utils.FileUtils; import org.apache.fluss.utils.FlussPaths; import org.apache.fluss.utils.IOUtils; import org.apache.fluss.utils.clock.Clock; @@ -229,7 +231,7 @@ public final class Replica { // null if table without pk or haven't become leader private volatile @Nullable KvTablet kvTablet; private volatile @Nullable CloseableRegistry closeableRegistryForKv; - private @Nullable PeriodicSnapshotManager kvSnapshotManager; + private volatile @Nullable PeriodicSnapshotManager kvSnapshotManager; /** * Server-wide {@link ScannerManager}. Active sessions for this bucket are closed in {@link @@ -323,6 +325,15 @@ private void registerMetrics() { logicalStorageMetrics.gauge(MetricNames.LOCAL_STORAGE_KV_SIZE, this::logicalStorageKvSize); } + private void stopSnapshotManager() { + PeriodicSnapshotManager mgr = this.kvSnapshotManager; + if (mgr != null) { + this.kvSnapshotManager = null; + closeableRegistryForKv.unregisterCloseable(mgr); + mgr.close(); + } + } + public long logicalStorageLogSize() { if (isLeader()) { return logTablet.logicalStorageSize(); @@ -340,8 +351,12 @@ public long logicalStorageKvSize() { KvTablet currentKvTablet = kvTablet; return currentKvTablet == null ? 0L : currentKvTablet.liveSstFilesSize(); } - checkNotNull(kvSnapshotManager, "kvSnapshotManager is null"); - return kvSnapshotManager.getSnapshotSize(); + PeriodicSnapshotManager snapshots = kvSnapshotManager; + if (snapshots != null) { + return snapshots.getSnapshotSize(); + } + KvTablet kv = kvTablet; + return kv != null && kv.isLazyMode() ? getLatestKvSnapshotSize() : 0L; } else { // follower doesn't need to report the logical storage size. return 0L; @@ -771,12 +786,21 @@ private void createKv() { e); } + checkNotNull(kvManager); + if (kvManager.isLazyOpenEnabled()) { + createKvLazy(); + } else { + createKvEager(); + } + } + + private void createKvEager() { // init kv tablet and get the snapshot it uses to init if have any Optional snapshotUsed = Optional.empty(); Exception lastError = null; for (int i = 1; i <= INIT_KV_TABLET_MAX_RETRY_TIMES; i++) { try { - snapshotUsed = initKvTablet(); + snapshotUsed = initKvTabletEager(); lastError = null; break; } catch (Exception e) { @@ -802,24 +826,31 @@ private void createKv() { lastError); } if (!isHistoricalPartition()) { - startPeriodicKvSnapshot(snapshotUsed.orElse(null)); + startPeriodicKvSnapshot(kvTablet, snapshotUsed.orElse(null)); } } private void dropKv() { - // Release scanner leases first; otherwise resourceGuard.close() inside kvTablet.close() - // blocks waiting for them. Runs under leaderIsrUpdateLock(W), so no concurrent register. scannerManager.closeScannersForBucket(tableBucket); - + // close any closeable registry for kv if (closeableRegistry.unregisterCloseable(closeableRegistryForKv)) { IOUtils.closeQuietly(closeableRegistryForKv); } - if (kvTablet != null) { - bucketMetricGroup.unregisterRocksDBStatistics(); - - checkNotNull(kvManager); - kvManager.dropKv(tableBucket); - kvTablet = null; + KvTablet kv = this.kvTablet; + if (kv != null) { + if (kv.isLazyMode()) { + // Delegate to KvTablet's lazy lifecycle which handles all states + // (dropCallback inside dropKvLazy already calls stopSnapshotManager) + kv.dropKvLazy(); + // Remove sentinel from KvManager registry + checkNotNull(kvManager); + kvManager.unregisterKv(tableBucket); + } else { + bucketMetricGroup.unregisterRocksDBStatistics(); + checkNotNull(kvManager); + kvManager.dropKv(tableBucket); + } + this.kvTablet = null; } } @@ -855,7 +886,131 @@ private void onKvFlushComplete() { * * @return the snapshot used to init kv tablet, empty if no any snapshot. */ - private Optional initKvTablet() { + private Optional initKvTabletEager() { + InitKvResult result = initKvTablet(); + kvManager.registerKv(tableBucket, result.tablet); + kvTablet = result.tablet; + if (kvTablet.getRocksDBStatistics() != null) { + bucketMetricGroup.registerRocksDBStatistics(kvTablet.getRocksDBStatistics()); + } + return result.snapshotUsed; + } + + private void createKvLazy() { + TableConfig tableConfig = getTableConfig(); + TabletServerMetricGroup serverMetrics = + bucketMetricGroup.getTableMetricGroup().getServerMetricGroup(); + + // Create a lazy sentinel — no RocksDB yet + KvTablet sentinel = + KvTablet.createLazySentinel(physicalPath, tableBucket, logTablet, serverMetrics); + + // Configure lazy open parameters via lifecycle + KvTabletLazyLifecycle lifecycle = sentinel.getLifecycle(); + lifecycle.configureLazyOpen( + clock, + kvManager.getOpenSemaphore(), + kvManager.getOpenTimeoutMs(), + kvManager.getFailedBackoffBaseMs(), + kvManager.getFailedBackoffMaxMs(), + KvManager.RELEASE_DRAIN_TIMEOUT_MS); + lifecycle.setLeaderEpochSupplier(() -> leaderEpoch); + lifecycle.setBucketEpochSupplier(() -> bucketEpoch); + + // Tablet directory supplier for LAZY/FAILED state cleanup + lifecycle.setTabletDirSupplier( + () -> + kvManager.getTabletDirPath( + logTablet.getDataDir(), physicalPath, tableBucket)); + + // Shared state between open and commit callbacks to pass snapshot reference + AtomicReference snapshotRef = new AtomicReference<>(); + + // Open callback: creates a real KvTablet + // If local data exists (reopen after idle release), load from local directory + // and replay only incremental logs. Otherwise, do full init (download snapshot). + lifecycle.setOpenCallback( + hasLocal -> { + if (hasLocal) { + return reopenFromLocalData(sentinel, snapshotRef); + } else { + InitKvResult result = initKvTablet(); + snapshotRef.set(result.snapshotUsed.orElse(null)); + return result.tablet; + } + }); + + // Commit callback: start periodic snapshot after RocksDB is installed + lifecycle.setCommitCallback( + kv -> { + if (!isHistoricalPartition()) { + startPeriodicKvSnapshot(kv, snapshotRef.getAndSet(null)); + } + if (kv.getRocksDBStatistics() != null) { + bucketMetricGroup.registerRocksDBStatistics(kv.getRocksDBStatistics()); + } + }); + + // Release callback: stop snapshot and unregister metrics + // RocksDB close is handled by detachRocksDB() inside KvTablet + lifecycle.setReleaseCallback( + kv -> { + try { + stopSnapshotManager(); + } catch (Exception e) { + LOG.warn( + "Failed to stop kv snapshot manager during release for {}", + tableBucket, + e); + } + try { + bucketMetricGroup.unregisterRocksDBStatistics(); + } catch (Exception e) { + LOG.warn( + "Failed to unregister RocksDB statistics during release for {}", + tableBucket, + e); + } + }); + + // Drop callback: cleanup metrics and snapshot + // Registry removal is handled by Replica.dropKv() + lifecycle.setDropCallback( + kv -> { + bucketMetricGroup.unregisterRocksDBStatistics(); + stopSnapshotManager(); + }); + + // Initialize cached row count from snapshot metadata if available (no RocksDB needed). + // This is best-effort — if ZK is unavailable, default to 0 rather than failing + // the entire lazy sentinel creation. + long initialRowCount; + if (isHistoricalPartition() || !supportsExactRowCount(tableConfig)) { + initialRowCount = -1L; + } else { + try { + Long snapshotRowCount = + getLatestSnapshot(tableBucket) + .map(CompletedSnapshot::getRowCount) + .orElse(null); + initialRowCount = snapshotRowCount != null ? snapshotRowCount : 0L; + } catch (Exception e) { + LOG.warn( + "Failed to read snapshot row count for {}, defaulting to 0", + tableBucket, + e); + initialRowCount = 0L; + } + } + lifecycle.initCachedRowCount(initialRowCount); + + // Register sentinel in KvManager registry + kvManager.registerKv(tableBucket, sentinel); + + this.kvTablet = sentinel; + } + + private InitKvResult initKvTablet() { checkNotNull(kvManager); TableConfig tableConfig = getTableConfig(); long startTime = clock.milliseconds(); @@ -881,6 +1036,7 @@ private Optional initKvTablet() { // never restore a normal KV snapshot, even if one exists from older code. Optional optCompletedSnapshot = isHistoricalPartition() ? Optional.empty() : getLatestSnapshot(tableBucket); + KvTablet createdTablet = null; try { Long rowCount; AutoIncIDRange autoIncIDRange; @@ -899,9 +1055,11 @@ private Optional initKvTablet() { downloadKvSnapshots(completedSnapshot, tabletDir.toPath()); // as we have downloaded kv files into the tablet dir, now, we can load it - kvTablet = kvManager.loadKv(tabletDir, schemaGetter, this::onKvFlushComplete); + createdTablet = + kvManager.loadKvUnregistered( + tabletDir, schemaGetter, this::onKvFlushComplete); - checkNotNull(kvTablet, "kv tablet should not be null."); + checkNotNull(createdTablet, "kv tablet should not be null."); restoreStartOffset = completedSnapshot.getLogOffset(); rowCount = supportsExactRowCount(tableConfig) ? completedSnapshot.getRowCount() : null; @@ -913,13 +1071,9 @@ private Optional initKvTablet() { tableBucket, physicalPath); - // Rebuild state in a fresh directory, as with snapshot recovery. - // Recovery retries can reuse the tablet opened by the first attempt. - if (kvTablet == null) { - kvManager.createTabletDir(logTablet.getDataDir(), physicalPath, tableBucket); - } - kvTablet = - kvManager.getOrCreateKv( + kvManager.createTabletDir(logTablet.getDataDir(), physicalPath, tableBucket); + createdTablet = + kvManager.createKvTabletUnregistered( physicalPath, tableBucket, logTablet, @@ -945,11 +1099,18 @@ private Optional initKvTablet() { logTablet.updateMinRetainOffset(restoreStartOffset); if (isHistoricalPartition()) { - checkNotNull(kvTablet, "kv tablet should not be null.") + checkNotNull(createdTablet, "kv tablet should not be null.") .advanceHistoricalCleanupOffset(restoreStartOffset); } - recoverKvTablet(restoreStartOffset, rowCount, autoIncIDRange); + recoverKvTablet(createdTablet, restoreStartOffset, rowCount, autoIncIDRange); } catch (Exception e) { + if (createdTablet != null) { + try { + createdTablet.close(); + } catch (Exception closeError) { + e.addSuppressed(closeError); + } + } throw new KvStorageException( String.format( "Fail to init kv tablet for %s of table %s.", @@ -963,14 +1124,90 @@ private Optional initKvTablet() { tableBucket, endTime - startTime); - if (kvTablet != null) { - // Register RocksDB statistics now that the kv tablet is fully initialized. - if (kvTablet.getRocksDBStatistics() != null) { - bucketMetricGroup.registerRocksDBStatistics(kvTablet.getRocksDBStatistics()); + return new InitKvResult(createdTablet, optCompletedSnapshot); + } + + private KvTablet reopenFromLocalData( + KvTablet sentinel, AtomicReference snapshotRef) { + checkNotNull(kvManager); + long startTime = clock.milliseconds(); + LOG.info( + "Reopening kv tablet from local data for {} of table {}.", + tableBucket, + physicalPath); + + long cachedFlushedLogOffset = sentinel.getCachedFlushedLogOffset(); + long cachedRowCount = sentinel.getCachedRowCount(); + + KvTablet createdTablet = null; + try { + File tabletDir = + kvManager.getTabletDirPath(logTablet.getDataDir(), physicalPath, tableBucket); + if (!new File(tabletDir, RocksDBKvBuilder.DB_INSTANCE_DIR_STRING).isDirectory()) { + throw new IOException("Local RocksDB directory is missing for " + tableBucket); + } + createdTablet = + kvManager.loadKvUnregistered(tabletDir, schemaGetter, this::onKvFlushComplete); + + if (isHistoricalPartition()) { + createdTablet.advanceHistoricalCleanupOffset( + logTablet.getLakeLogEndOffset() >= 0 + ? logTablet.getLakeLogEndOffset() + : 0L); + } + Long rowCount = cachedRowCount == KvTablet.ROW_COUNT_DISABLED ? null : cachedRowCount; + logTablet.updateMinRetainOffset(cachedFlushedLogOffset); + recoverKvTablet(createdTablet, cachedFlushedLogOffset, rowCount, null); + + long endTime = clock.milliseconds(); + LOG.info( + "Reopened kv tablet from local data for {} of table {} " + + "from log offset {}, cost {} ms.", + tableBucket, + physicalPath, + cachedFlushedLogOffset, + endTime - startTime); + + return createdTablet; + } catch (Exception e) { + LOG.warn( + "Failed to reopen from local data for {} of table {}, " + + "falling back to full init.", + tableBucket, + physicalPath, + e); + if (createdTablet != null) { + try { + createdTablet.close(); + } catch (Exception closeEx) { + e.addSuppressed(closeEx); + } } + // Clean up the potentially corrupted local directory + File tabletDir = + kvManager.getTabletDirPath(logTablet.getDataDir(), physicalPath, tableBucket); + FileUtils.deleteDirectoryQuietly(tabletDir); + if (tabletDir.exists()) { + LOG.error( + "Failed to delete corrupted tablet dir {} during reopen fallback", + tabletDir.getAbsolutePath()); + } + // Fall back to full init (download snapshot) + InitKvResult result = initKvTablet(); + snapshotRef.set(result.snapshotUsed.orElse(null)); + return result.tablet; } + } + + /** Result of {@link #initKvTablet()}: the created tablet and the snapshot used (if any). */ + private static class InitKvResult { + final KvTablet tablet; + final Optional snapshotUsed; - return optCompletedSnapshot; + InitKvResult(KvTablet tablet, Optional snapshotUsed) { + this.tablet = tablet; + this.snapshotUsed = snapshotUsed; + } } private void downloadKvSnapshots(CompletedSnapshot completedSnapshot, Path kvTabletDir) @@ -1037,12 +1274,13 @@ private Optional getLatestSnapshot(TableBucket tableBucket) { } private void recoverKvTablet( + KvTablet tablet, long startRecoverLogOffset, @Nullable Long rowCount, @Nullable AutoIncIDRange autoIncIDRange) { TableConfig tableConfig = getTableConfig(); long start = clock.milliseconds(); - checkNotNull(kvTablet, "kv tablet should not be null."); + checkNotNull(tablet, "kv tablet should not be null."); try { KvRecoverHelper.KvRecoverContext recoverContext = new KvRecoverHelper.KvRecoverContext( @@ -1062,7 +1300,7 @@ private void recoverKvTablet( try { KvRecoverHelper kvRecoverHelper = new KvRecoverHelper( - kvTablet, + tablet, logTablet, startRecoverLogOffset, rowCount, @@ -1113,7 +1351,8 @@ private long historicalRecoveryStartOffset() { return recoveryStartOffset; } - private void startPeriodicKvSnapshot(@Nullable CompletedSnapshot completedSnapshot) { + private void startPeriodicKvSnapshot( + KvTablet kvTablet, @Nullable CompletedSnapshot completedSnapshot) { checkNotNull(kvTablet); KvTabletSnapshotTarget kvTabletSnapshotTarget; try { @@ -1198,11 +1437,12 @@ private void startPeriodicKvSnapshot(@Nullable CompletedSnapshot completedSnapsh } public long getLatestKvSnapshotSize() { - if (kvSnapshotManager == null) { - return 0L; - } else { - return kvSnapshotManager.getSnapshotSize(); + PeriodicSnapshotManager snapshots = kvSnapshotManager; + if (snapshots != null) { + return snapshots.getSnapshotSize(); } + CompletedSnapshot latest = getLatestSnapshot(tableBucket).orElse(null); + return latest == null ? 0L : latest.getSnapshotSize(); } public long getLeaderEndOffsetSnapshot() { @@ -1277,9 +1517,11 @@ public LogAppendInfo putRecordsToLeader( MergeMode mergeMode, int requiredAcks) throws Exception { - return inReadLock( - leaderIsrUpdateLock, - () -> { + return withGuardedLeaderKv( + true, + requiredAcks, + "KvTablet for leader access must not be null.", + kv -> { if (!isLeader()) { throw new NotLeaderOrFollowerException( String.format( @@ -1292,7 +1534,6 @@ public LogAppendInfo putRecordsToLeader( } validateInSyncReplicaSize(requiredAcks); - KvTablet kv = this.kvTablet; checkNotNull( kv, "KvTablet for the replica to put kv records shouldn't be null."); LogAppendInfo logAppendInfo; @@ -1324,11 +1565,12 @@ public LogAppendInfo putHistoricalRecordsToLeader( int requiredAcks) throws Exception { LocalValueLookupResult localLookupResult = - inReadLock( - leaderIsrUpdateLock, - () -> { + withGuardedLeaderKv( + true, + requiredAcks, + "Historical tablet must not be null.", + kv -> { validateHistoricalWrite(expectedLeaderEpoch, requiredAcks); - KvTablet kv = this.kvTablet; checkNotNull( kv, "KvTablet for the historical replica shouldn't be null."); return kv.probeLocalPreviousValues( @@ -1339,11 +1581,12 @@ public LogAppendInfo putHistoricalRecordsToLeader( HistoricalValueLookup historicalValueLookup = localLookupResult.createValueLookup(lakeLookup); - return inReadLock( - leaderIsrUpdateLock, - () -> { + return withGuardedLeaderKv( + true, + requiredAcks, + "Historical tablet must not be null.", + kv -> { validateHistoricalWrite(expectedLeaderEpoch, requiredAcks); - KvTablet kv = this.kvTablet; checkNotNull(kv, "KvTablet for the historical replica shouldn't be null."); LogAppendInfo appendInfo = kv.putHistoricalAsLeader( @@ -1418,9 +1661,11 @@ private void validateHistoricalWrite(int expectedLeaderEpoch, int requiredAcks) /** Looks up keys from the local historical KV state of the leader replica. */ public List lookupHistoricalLocal( String originalPartitionName, List keys) throws Exception { - return inReadLock( - leaderIsrUpdateLock, - () -> { + return withGuardedLeaderKv( + false, + 0, + "KvTablet for leader access must not be null.", + kv -> { if (!isLeader()) { throw new NotLeaderOrFollowerException( String.format( @@ -1432,7 +1677,6 @@ public List lookupHistoricalLocal( "Historical lookup request must target a historical partition."); } - KvTablet kv = this.kvTablet; checkNotNull(kv, "KvTablet for the historical replica shouldn't be null."); List results = new ArrayList<>(keys.size()); for (byte[] key : keys) { @@ -1659,9 +1903,11 @@ public List lookups(List keys) { throw new NonPrimaryKeyTableException( "the primary key table not exists for " + tableBucket); } - return inReadLock( - leaderIsrUpdateLock, - () -> { + return withGuardedLeaderKv( + false, + 0, + "KvTablet for leader access must not be null.", + kv -> { try { if (!isLeader()) { throw new NotLeaderOrFollowerException( @@ -1671,7 +1917,7 @@ public List lookups(List keys) { } checkNotNull( kvTablet, "KvTablet for the replica to get key shouldn't be null."); - return kvTablet.multiGet(keys); + return kv.multiGet(keys); } catch (IOException e) { String errorMsg = String.format( @@ -1694,9 +1940,11 @@ public List lookupsFromBufferOrKv(List keys) { throw new NonPrimaryKeyTableException( "the primary key table not exists for " + tableBucket); } - return inReadLock( - leaderIsrUpdateLock, - () -> { + return withGuardedLeaderKv( + false, + 0, + "KvTablet for leader access must not be null.", + kv -> { try { if (!isLeader()) { throw new NotLeaderOrFollowerException( @@ -1706,7 +1954,7 @@ public List lookupsFromBufferOrKv(List keys) { } checkNotNull( kvTablet, "KvTablet for the replica to get key shouldn't be null."); - return kvTablet.multiGetFromBufferOrKv(keys); + return kv.multiGetFromBufferOrKv(keys); } catch (IOException e) { String errorMsg = String.format( @@ -1724,9 +1972,11 @@ public List prefixLookup(byte[] prefixKey) { "Try to do prefix lookup on a non primary key table: " + getTablePath()); } - return inReadLock( - leaderIsrUpdateLock, - () -> { + return withGuardedLeaderKv( + false, + 0, + "KvTablet for leader access must not be null.", + kv -> { try { if (!isLeader()) { throw new NotLeaderOrFollowerException( @@ -1736,7 +1986,7 @@ public List prefixLookup(byte[] prefixKey) { } checkNotNull( kvTablet, "KvTablet for the replica to get key shouldn't be null."); - return kvTablet.prefixLookup(prefixKey); + return kv.prefixLookup(prefixKey); } catch (IOException e) { String errorMsg = String.format( @@ -1754,9 +2004,11 @@ public DefaultValueRecordBatch limitKvScan(int limit) { "the primary key table not exists for " + tableBucket); } - return inReadLock( - leaderIsrUpdateLock, - () -> { + return withGuardedLeaderKv( + false, + 0, + "KvTablet for leader access must not be null.", + kv -> { try { if (!isLeader()) { throw new NotLeaderOrFollowerException( @@ -1767,7 +2019,7 @@ public DefaultValueRecordBatch limitKvScan(int limit) { checkNotNull( kvTablet, "KvTablet for the replica to limit scan shouldn't be null."); - List values = kvTablet.limitScan(limit); + List values = kv.limitScan(limit); DefaultValueRecordBatch.Builder builder = DefaultValueRecordBatch.builder(); for (ByteArraySlice value : values) { builder.append(value.array(), value.offset(), value.length()); @@ -1803,9 +2055,11 @@ public OpenScanResult openScan( "the primary key table not exists for " + tableBucket); } - return inReadLock( - leaderIsrUpdateLock, - () -> { + return withGuardedLeaderKv( + false, + 0, + "KvTablet for leader access must not be null.", + kv -> { if (!isLeader()) { throw new NotLeaderOrFollowerException( String.format( @@ -1814,8 +2068,7 @@ public OpenScanResult openScan( } checkNotNull( kvTablet, "KvTablet for the replica to open scan shouldn't be null."); - OpenScanResult result = - kvTablet.openScan(scannerId, limit, initialAccessTimeMs); + OpenScanResult result = kv.openScan(scannerId, limit, initialAccessTimeMs); ScannerContext context = result.getContext(); if (context == null) { return result; @@ -1830,6 +2083,67 @@ public OpenScanResult openScan( }); } + private T withGuardedLeaderKv( + boolean validateInSyncReplicas, + int requiredAcks, + String nullKvMessage, + GuardedKvOperation action) + throws E { + while (true) { + KvTablet kv = + inReadLock( + leaderIsrUpdateLock, + () -> { + ensureLeaderForKvAccess(); + if (validateInSyncReplicas) { + validateInSyncReplicaSize(requiredAcks); + } + return checkNotNull(kvTablet, nullKvMessage); + }); + boolean openingKv = kv.isLazyMode() && !kv.isLazyOpen(); + try (KvTablet.Guard ignored = kv.acquireGuard()) { + // A leadership transition drains pins while holding the write lock. Never wait + // for that lock with a pin: release it and retry after the transition instead. + if (leaderIsrUpdateLock.readLock().tryLock()) { + try { + ensureLeaderForKvAccess(); + if (this.kvTablet != kv) { + throw new NotLeaderOrFollowerException( + "KV tablet changed while opening " + tableBucket); + } + if (validateInSyncReplicas) { + validateInSyncReplicaSize(requiredAcks); + } + return action.apply(kv); + } finally { + leaderIsrUpdateLock.readLock().unlock(); + } + } + } finally { + // Recovery can leave WAL entries above the old high watermark in the buffer. + // Publish progress only after releasing the pin and replica read lock. + if (openingKv && kv.isLazyOpen()) { + onKvFlushComplete(); + } + } + inReadLock(leaderIsrUpdateLock, this::ensureLeaderForKvAccess); + } + } + + private void ensureLeaderForKvAccess() { + if (!isLeader()) { + throw new NotLeaderOrFollowerException( + String.format( + "Leader not local for bucket %s on tabletServer %d", + tableBucket, localTabletServerId)); + } + } + + @FunctionalInterface + private interface GuardedKvOperation { + T apply(KvTablet tablet) throws E; + } + public LogRecords limitLogScan(int limit) { return inReadLock( leaderIsrUpdateLock, @@ -1901,18 +2215,15 @@ public Tuple2 checkEnoughReplicasReachOffset(long requiredOffse } public long getRowCount() { - return inReadLock( - leaderIsrUpdateLock, - () -> { - KvTablet kv = this.kvTablet; - if (kv != null) { - // return materialized row count for primary key table - return kv.getRowCount(); - } else { - // return log row count for non-primary key table - return logTablet.getRowCount(); - } - }); + KvTablet kv = inReadLock(leaderIsrUpdateLock, () -> kvTablet); + + if (kv != null) { + return kv.getRowCount(); + } else if (isKvTable()) { + return 0L; + } else { + return logTablet.getRowCount(); + } } private static boolean supportsExactRowCount(TableConfig tableConfig) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index 41ce69fce7a..3ac4715f3bd 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -88,8 +88,10 @@ import org.apache.fluss.server.entity.StopReplicaData; import org.apache.fluss.server.entity.StopReplicaResultForBucket; import org.apache.fluss.server.entity.UserContext; +import org.apache.fluss.server.kv.KvIdleReleaseController; import org.apache.fluss.server.kv.KvManager; import org.apache.fluss.server.kv.KvSnapshotResource; +import org.apache.fluss.server.kv.KvTablet; import org.apache.fluss.server.kv.scan.ScannerManager; import org.apache.fluss.server.kv.snapshot.CompletedKvSnapshotCommitter; import org.apache.fluss.server.kv.snapshot.DefaultSnapshotContext; @@ -125,9 +127,11 @@ import org.apache.fluss.server.zk.data.LeaderAndIsr; import org.apache.fluss.server.zk.data.lake.LakeTableSnapshot; import org.apache.fluss.utils.ByteArraySlice; +import org.apache.fluss.utils.ExecutorUtils; import org.apache.fluss.utils.FileUtils; import org.apache.fluss.utils.FlussPaths; import org.apache.fluss.utils.clock.Clock; +import org.apache.fluss.utils.concurrent.ExecutorThreadFactory; import org.apache.fluss.utils.concurrent.FutureUtils; import org.apache.fluss.utils.concurrent.Scheduler; @@ -150,11 +154,15 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; @@ -183,6 +191,8 @@ public class ReplicaManager implements ServerReconfigurable { * safe. */ private static final short PUT_KV_VERSION_WITH_STORAGE_BACKPRESSURE = 2; + // ---- Idle release internal constants ---- + private static final long IDLE_RELEASE_CHECK_INTERVAL_MS = 60_000; private final Configuration conf; private final Scheduler scheduler; @@ -244,6 +254,9 @@ public class ReplicaManager implements ServerReconfigurable { private final ScannerManager scannerManager; private final HistoricalPartitionManager historicalPartitionManager; + private final boolean kvIdleReleaseEnabled; + private @Nullable KvIdleReleaseController kvIdleReleaseController; + private @Nullable ScheduledExecutorService kvIdleReleaseScheduler; public ReplicaManager( Configuration conf, @@ -378,6 +391,30 @@ public ReplicaManager( dataDirVolumeBytes, scheduler); + this.kvIdleReleaseEnabled = kvManager != null && kvManager.isLazyOpenEnabled(); + if (kvIdleReleaseEnabled) { + long closeIdleIntervalMs = conf.get(ConfigOptions.KV_LAZY_OPEN_IDLE_TIMEOUT).toMillis(); + checkArgument( + closeIdleIntervalMs > 0, + "kv.lazy-open.idle-timeout must be positive, got: %s ms", + closeIdleIntervalMs); + this.kvIdleReleaseScheduler = + Executors.newSingleThreadScheduledExecutor( + new ExecutorThreadFactory("kv-idle-release")); + this.kvIdleReleaseController = + new KvIdleReleaseController( + kvIdleReleaseScheduler, + () -> + onlineReplicas() + .filter(Replica::isKvTable) + .map(Replica::getKvTablet) + .filter(Objects::nonNull) + .filter(KvTablet::isLazyOpen) + .collect(Collectors.toList()), + clock, + IDLE_RELEASE_CHECK_INTERVAL_MS, + closeIdleIntervalMs); + } registerMetrics(); } @@ -395,6 +432,10 @@ public void startup() { // Start periodic disk usage monitoring (initial + periodic sampling) localDiskManager.startDiskUsageMonitor(scheduler); + // Start idle release controller if enabled + if (kvIdleReleaseController != null) { + kvIdleReleaseController.start(); + } } public RemoteLogManager getRemoteLogManager() { @@ -2677,6 +2718,14 @@ public Replica getReplica() { public static final class OfflineReplica implements HostedReplica {} public void shutdown() throws InterruptedException { + // Shut down the idle release controller if it was created + if (kvIdleReleaseController != null) { + kvIdleReleaseController.close(); + } + if (kvIdleReleaseScheduler != null) { + ExecutorUtils.gracefulShutdown(5, TimeUnit.SECONDS, kvIdleReleaseScheduler); + } + // Close the resources for snapshot kv kvSnapshotResource.close(); historicalPartitionManager.close(); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvIdleReleaseControllerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvIdleReleaseControllerTest.java new file mode 100644 index 00000000000..8909d059359 --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvIdleReleaseControllerTest.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.kv; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.TableConfig; +import org.apache.fluss.metadata.KvFormat; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.PhysicalTablePath; +import org.apache.fluss.metadata.SchemaInfo; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.KvRecord; +import org.apache.fluss.record.KvRecordBatch; +import org.apache.fluss.record.KvRecordTestUtils; +import org.apache.fluss.record.TestData; +import org.apache.fluss.record.TestingSchemaGetter; +import org.apache.fluss.server.log.LogManager; +import org.apache.fluss.server.log.LogTablet; +import org.apache.fluss.server.metrics.group.TestingMetricGroups; +import org.apache.fluss.server.storage.LocalDiskManager; +import org.apache.fluss.server.zk.NOPErrorHandler; +import org.apache.fluss.server.zk.ZooKeeperClient; +import org.apache.fluss.server.zk.ZooKeeperExtension; +import org.apache.fluss.testutils.common.AllCallbackWrapper; +import org.apache.fluss.testutils.common.CommonTestUtils; +import org.apache.fluss.testutils.common.ManuallyTriggeredScheduledExecutorService; +import org.apache.fluss.types.RowType; +import org.apache.fluss.utils.clock.ManualClock; +import org.apache.fluss.utils.clock.SystemClock; +import org.apache.fluss.utils.concurrent.FlussScheduler; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.apache.fluss.compression.ArrowCompressionInfo.DEFAULT_COMPRESSION; +import static org.apache.fluss.record.TestData.DATA1_SCHEMA_PK; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; + +/** Test for {@link KvIdleReleaseController}. */ +final class KvIdleReleaseControllerTest { + + @RegisterExtension + public static final AllCallbackWrapper ZOO_KEEPER_EXTENSION_WRAPPER = + new AllCallbackWrapper<>(new ZooKeeperExtension()); + + private static final short SCHEMA_ID = 1; + private static final RowType BASE_ROW_TYPE = TestData.DATA1_ROW_TYPE; + private static final long IDLE_INTERVAL_MS = 60_000; + private static final long CHECK_INTERVAL_MS = 10_000; + + private final KvRecordTestUtils.KvRecordBatchFactory batchFactory = + KvRecordTestUtils.KvRecordBatchFactory.of(SCHEMA_ID); + private final KvRecordTestUtils.KvRecordFactory recordFactory = + KvRecordTestUtils.KvRecordFactory.of(BASE_ROW_TYPE); + + private static ZooKeeperClient zkClient; + + private @TempDir File tempDir; + + private LocalDiskManager localDiskManager; + private LogManager logManager; + private KvManager kvManager; + private Configuration conf; + private ManualClock clock; + private ManuallyTriggeredScheduledExecutorService scheduler; + private List allTablets; + private int nextTableId = 30001; + + @BeforeAll + static void baseBeforeAll() { + zkClient = + ZOO_KEEPER_EXTENSION_WRAPPER + .getCustomExtension() + .getZooKeeperClient(NOPErrorHandler.INSTANCE); + } + + @BeforeEach + void setup() throws Exception { + conf = new Configuration(); + conf.setString(ConfigOptions.DATA_DIR, tempDir.getAbsolutePath()); + conf.set(ConfigOptions.KV_LAZY_OPEN_ENABLED, true); + + conf.set(ConfigOptions.TABLET_SERVER_ID, 1); + localDiskManager = LocalDiskManager.create(conf); + logManager = + LogManager.create( + conf, + zkClient, + new FlussScheduler(1), + SystemClock.getInstance(), + TestingMetricGroups.TABLET_SERVER_METRICS, + localDiskManager); + kvManager = + KvManager.create( + conf, + zkClient, + logManager, + TestingMetricGroups.TABLET_SERVER_METRICS, + localDiskManager); + kvManager.startup(); + + clock = new ManualClock(System.currentTimeMillis()); + scheduler = new ManuallyTriggeredScheduledExecutorService(); + allTablets = new ArrayList<>(); + } + + @AfterEach + void tearDown() throws Exception { + if (kvManager != null) { + kvManager.shutdown(); + } + if (logManager != null) { + logManager.shutdown(); + } + if (localDiskManager != null) { + localDiskManager.close(); + } + } + + // ---- Helper methods ---- + + private KvTablet createAndOpenTablet(int tableId, int bucketId) throws Exception { + TablePath tablePath = TablePath.of("db1", "t_idle_" + tableId); + PhysicalTablePath physicalPath = + PhysicalTablePath.of(tablePath.getDatabaseName(), tablePath.getTableName(), null); + TableBucket tableBucket = new TableBucket((long) tableId, bucketId); + LogTablet logTablet = + logManager.getOrCreateLog( + tempDir, physicalPath, tableBucket, LogFormat.ARROW, 1, true); + + KvTablet sentinel = + KvTablet.createLazySentinel( + physicalPath, + tableBucket, + logTablet, + TestingMetricGroups.TABLET_SERVER_METRICS); + + KvTabletLazyLifecycle lifecycle = sentinel.getLifecycle(); + lifecycle.configureLazyOpen( + clock, + kvManager.getOpenSemaphore(), + kvManager.getOpenTimeoutMs(), + kvManager.getFailedBackoffBaseMs(), + kvManager.getFailedBackoffMaxMs(), + KvManager.RELEASE_DRAIN_TIMEOUT_MS); + lifecycle.setLeaderEpochSupplier(() -> 1); + lifecycle.setBucketEpochSupplier(() -> 1); + lifecycle.setTabletDirSupplier( + () -> + kvManager.getTabletDirPath( + logTablet.getDataDir(), physicalPath, tableBucket)); + lifecycle.setOpenCallback( + hasLocal -> + kvManager.createKvTabletUnregistered( + physicalPath, + tableBucket, + logTablet, + KvFormat.COMPACTED, + new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), + new TableConfig(new Configuration()), + DEFAULT_COMPRESSION, + null)); + lifecycle.setCommitCallback(kv -> {}); + lifecycle.setReleaseCallback(kv -> {}); + lifecycle.setDropCallback(kv -> {}); + + // Open the tablet via acquireGuard + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + // Write and flush so releaseKv() won't be rejected + KvRecord record = + recordFactory.ofRecord( + ("k_" + tableId + "_" + bucketId).getBytes(), + new Object[] {tableId, "v"}); + KvRecordBatch batch = batchFactory.ofRecords(Collections.singletonList(record)); + sentinel.putAsLeader(batch, null); + long offset = logTablet.localLogEndOffset(); + sentinel.requestFlush(offset, NOPErrorHandler.INSTANCE); + CommonTestUtils.waitUntil( + () -> sentinel.getFlushedLogOffset() >= offset, + Duration.ofSeconds(10), + "KV operation did not complete"); + } + + kvManager.registerKv(tableBucket, sentinel); + return sentinel; + } + + private Supplier> openTabletSupplier() { + return () -> allTablets.stream().filter(KvTablet::isLazyOpen).collect(Collectors.toList()); + } + + private KvIdleReleaseController createController() { + return new KvIdleReleaseController( + scheduler, openTabletSupplier(), clock, CHECK_INTERVAL_MS, IDLE_INTERVAL_MS); + } + + // ---- Tests ---- + + @Test + void testNoReleaseWhenAllActive() throws Exception { + KvTablet tablet1 = createAndOpenTablet(nextTableId++, 0); + KvTablet tablet2 = createAndOpenTablet(nextTableId++, 0); + allTablets.add(tablet1); + allTablets.add(tablet2); + + // Don't advance clock — tablets are recently accessed + KvIdleReleaseController controller = createController(); + controller.checkAndRelease(); + + assertThat(tablet1.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + assertThat(tablet2.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + } + + @Test + void testReleaseAllIdleTablets() throws Exception { + KvTablet tablet1 = createAndOpenTablet(nextTableId++, 0); + KvTablet tablet2 = createAndOpenTablet(nextTableId++, 0); + allTablets.add(tablet1); + allTablets.add(tablet2); + + // Advance clock past idle interval + clock.advanceTime(IDLE_INTERVAL_MS + 1, TimeUnit.MILLISECONDS); + + KvIdleReleaseController controller = createController(); + controller.checkAndRelease(); + + assertThat(tablet1.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); + assertThat(tablet2.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); + } + + @Test + void testOnlyIdleTabletsReleased() throws Exception { + // Create tablet1 first, then advance clock, then create tablet2 + KvTablet tablet1 = createAndOpenTablet(nextTableId++, 0); + allTablets.add(tablet1); + + // Advance clock so tablet1 becomes idle but tablet2 (created after) won't be + clock.advanceTime(IDLE_INTERVAL_MS + 1, TimeUnit.MILLISECONDS); + + KvTablet tablet2 = createAndOpenTablet(nextTableId++, 0); + allTablets.add(tablet2); + + KvIdleReleaseController controller = createController(); + controller.checkAndRelease(); + + // tablet1 is idle — released + assertThat(tablet1.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); + // tablet2 is recent — stays open + assertThat(tablet2.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + } + + @Test + void testSkipTabletWithActivePins() throws Exception { + KvTablet tablet1 = createAndOpenTablet(nextTableId++, 0); + KvTablet tablet2 = createAndOpenTablet(nextTableId++, 0); + allTablets.add(tablet1); + allTablets.add(tablet2); + + // Hold a guard on tablet1 (don't close it) — this pins it + try (KvTablet.Guard heldGuard = tablet1.acquireGuard()) { + // Advance clock past idle + clock.advanceTime(IDLE_INTERVAL_MS + 1, TimeUnit.MILLISECONDS); + + KvIdleReleaseController controller = createController(); + controller.checkAndRelease(); + + // tablet1 is pinned — should NOT be released + assertThat(tablet1.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + // tablet2 is idle and unpinned — should be released + assertThat(tablet2.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); + } + } + + @Test + void testContinuesAfterReleaseFailure() throws Exception { + KvTablet tablet1 = createAndOpenTablet(nextTableId++, 0); + KvTablet tablet2 = createAndOpenTablet(nextTableId++, 0); + allTablets.add(tablet1); + allTablets.add(tablet2); + + // Make tablet1's releaseCallback throw so releaseKv() returns false + tablet1.getLifecycle() + .setReleaseCallback( + kv -> { + throw new RuntimeException("simulated release failure"); + }); + + // Advance clock past idle + clock.advanceTime(IDLE_INTERVAL_MS + 1, TimeUnit.MILLISECONDS); + + KvIdleReleaseController controller = createController(); + assertThatCode(() -> controller.checkAndRelease()).doesNotThrowAnyException(); + + // tablet1 release failed — should rollback to OPEN + assertThat(tablet1.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + // tablet2 should still be released despite tablet1's failure + assertThat(tablet2.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvManagerTest.java index c153bfbfaf9..730cdc111ef 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvManagerTest.java @@ -783,6 +783,129 @@ void testDropKv(String partitionName) throws Exception { assertThat(kvManager.getKv(tableBucket1)).isPresent(); } + @ParameterizedTest + @MethodSource("partitionProvider") + void testRegisterKv(String partitionName) throws Exception { + initTableBuckets(partitionName); + PhysicalTablePath physicalTablePath = + PhysicalTablePath.of( + tablePath1.getDatabaseName(), tablePath1.getTableName(), partitionName); + LogTablet logTablet = + logManager.getOrCreateLog( + tempDir, physicalTablePath, tableBucket1, LogFormat.ARROW, 1, true); + + // Create a lazy sentinel and register it + KvTablet sentinel = + KvTablet.createLazySentinel(physicalTablePath, tableBucket1, logTablet, null); + kvManager.registerKv(tableBucket1, sentinel); + + // verify registered + assertThat(kvManager.getKv(tableBucket1)).isPresent(); + assertThat(kvManager.getKv(tableBucket1).get()).isSameAs(sentinel); + } + + @Test + void testRegisterKvDuplicate() throws Exception { + initTableBuckets(null); + PhysicalTablePath physicalTablePath = + PhysicalTablePath.of(tablePath1.getDatabaseName(), tablePath1.getTableName(), null); + LogTablet logTablet = + logManager.getOrCreateLog( + tempDir, physicalTablePath, tableBucket1, LogFormat.ARROW, 1, true); + + KvTablet sentinel = + KvTablet.createLazySentinel(physicalTablePath, tableBucket1, logTablet, null); + kvManager.registerKv(tableBucket1, sentinel); + + // registering again should throw + KvTablet sentinel2 = + KvTablet.createLazySentinel(physicalTablePath, tableBucket1, logTablet, null); + org.assertj.core.api.Assertions.assertThatThrownBy( + () -> kvManager.registerKv(tableBucket1, sentinel2)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("already registered"); + } + + @ParameterizedTest + @MethodSource("partitionProvider") + void testUnregisterKv(String partitionName) throws Exception { + initTableBuckets(partitionName); + KvTablet tablet = getOrCreateKv(tablePath1, partitionName, tableBucket1); + assertThat(kvManager.getKv(tableBucket1)).isPresent(); + + kvManager.unregisterKv(tableBucket1); + assertThat(kvManager.getKv(tableBucket1)).isNotPresent(); + tablet.close(); + } + + @ParameterizedTest + @MethodSource("partitionProvider") + void testCreateKvTabletUnregistered(String partitionName) throws Exception { + initTableBuckets(partitionName); + PhysicalTablePath physicalTablePath = + PhysicalTablePath.of( + tablePath1.getDatabaseName(), tablePath1.getTableName(), partitionName); + LogTablet logTablet = + logManager.getOrCreateLog( + tempDir, physicalTablePath, tableBucket1, LogFormat.ARROW, 1, true); + + KvTablet tablet = + kvManager.createKvTabletUnregistered( + physicalTablePath, + tableBucket1, + logTablet, + KvFormat.COMPACTED, + new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, 1)), + new TableConfig(new Configuration()), + DEFAULT_COMPRESSION, + null); + + // tablet should be created but NOT registered + assertThat(tablet.getKvTabletDir()).exists(); + assertThat(kvManager.getKv(tableBucket1)).isNotPresent(); + tablet.close(); + } + + @ParameterizedTest + @MethodSource("partitionProvider") + void testGetTabletDirPath(String partitionName) throws Exception { + initTableBuckets(partitionName); + PhysicalTablePath physicalTablePath = + PhysicalTablePath.of( + tablePath1.getDatabaseName(), tablePath1.getTableName(), partitionName); + + File dirPath = kvManager.getTabletDirPath(tempDir, physicalTablePath, tableBucket1); + // directory should not be created by this call + assertThat(dirPath).doesNotExist(); + + // Assert the exact expected path structure + File expectedDir; + if (partitionName == null) { + // Non-partitioned: {dataDir}/{db}/{table}-{tableId}/kv-{bucket} + expectedDir = + new File( + tempDir, + "db1/t1-" + + tableBucket1.getTableId() + + "/kv-" + + tableBucket1.getBucket()); + } else { + // Partitioned: {dataDir}/{db}/{table}-{tableId}/{partition}-p{partitionId}/kv-{bucket} + expectedDir = + new File( + tempDir, + "db1/t1-" + + tableBucket1.getTableId() + + "/" + + partitionName + + "-p" + + tableBucket1.getPartitionId() + + "/kv-" + + tableBucket1.getBucket()); + } + assertThat(dirPath).isEqualTo(expectedDir); + } + @Test void testKvRunConcurrentlyForDifferentBuckets() throws Exception { initTableBuckets(null); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletLazyLifecycleTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletLazyLifecycleTest.java new file mode 100644 index 00000000000..b46e5add3aa --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletLazyLifecycleTest.java @@ -0,0 +1,1295 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.kv; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.TableConfig; +import org.apache.fluss.exception.KvStorageException; +import org.apache.fluss.metadata.KvFormat; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.PhysicalTablePath; +import org.apache.fluss.metadata.SchemaInfo; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.record.KvRecord; +import org.apache.fluss.record.KvRecordBatch; +import org.apache.fluss.record.KvRecordTestUtils; +import org.apache.fluss.record.TestData; +import org.apache.fluss.record.TestingSchemaGetter; +import org.apache.fluss.row.encode.ValueEncoder; +import org.apache.fluss.server.log.LogManager; +import org.apache.fluss.server.log.LogTablet; +import org.apache.fluss.server.metrics.group.TabletServerMetricGroup; +import org.apache.fluss.server.metrics.group.TestingMetricGroups; +import org.apache.fluss.server.storage.LocalDiskManager; +import org.apache.fluss.server.zk.NOPErrorHandler; +import org.apache.fluss.server.zk.ZooKeeperClient; +import org.apache.fluss.server.zk.ZooKeeperExtension; +import org.apache.fluss.testutils.common.AllCallbackWrapper; +import org.apache.fluss.testutils.common.CommonTestUtils; +import org.apache.fluss.types.RowType; +import org.apache.fluss.utils.ByteArraySlice; +import org.apache.fluss.utils.clock.ManualClock; +import org.apache.fluss.utils.clock.SystemClock; +import org.apache.fluss.utils.concurrent.FlussScheduler; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.apache.fluss.compression.ArrowCompressionInfo.DEFAULT_COMPRESSION; +import static org.apache.fluss.record.TestData.DATA1_SCHEMA_PK; +import static org.apache.fluss.testutils.DataTestUtils.compactedRow; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Test for {@link KvTabletLazyLifecycle}. */ +final class KvTabletLazyLifecycleTest { + + @RegisterExtension + public static final AllCallbackWrapper ZOO_KEEPER_EXTENSION_WRAPPER = + new AllCallbackWrapper<>(new ZooKeeperExtension()); + + private static final short SCHEMA_ID = 1; + private static final RowType BASE_ROW_TYPE = TestData.DATA1_ROW_TYPE; + + private final KvRecordTestUtils.KvRecordBatchFactory batchFactory = + KvRecordTestUtils.KvRecordBatchFactory.of(SCHEMA_ID); + private final KvRecordTestUtils.KvRecordFactory recordFactory = + KvRecordTestUtils.KvRecordFactory.of(BASE_ROW_TYPE); + + private static ZooKeeperClient zkClient; + + private @TempDir File tempDir; + + private LocalDiskManager localDiskManager; + private LogManager logManager; + private KvManager kvManager; + private Configuration conf; + + private PhysicalTablePath physicalPath; + private TableBucket tableBucket; + private LogTablet logTablet; + + @BeforeAll + static void baseBeforeAll() { + zkClient = + ZOO_KEEPER_EXTENSION_WRAPPER + .getCustomExtension() + .getZooKeeperClient(NOPErrorHandler.INSTANCE); + } + + @BeforeEach + void setup() throws Exception { + conf = new Configuration(); + conf.setString(ConfigOptions.DATA_DIR, tempDir.getAbsolutePath()); + conf.set(ConfigOptions.KV_LAZY_OPEN_ENABLED, true); + + conf.set(ConfigOptions.TABLET_SERVER_ID, 1); + localDiskManager = LocalDiskManager.create(conf); + logManager = + LogManager.create( + conf, + zkClient, + new FlussScheduler(1), + SystemClock.getInstance(), + TestingMetricGroups.TABLET_SERVER_METRICS, + localDiskManager); + kvManager = + KvManager.create( + conf, + zkClient, + logManager, + TestingMetricGroups.TABLET_SERVER_METRICS, + localDiskManager); + kvManager.startup(); + + TablePath tablePath = TablePath.of("db1", "t_lazy"); + physicalPath = + PhysicalTablePath.of(tablePath.getDatabaseName(), tablePath.getTableName(), null); + tableBucket = new TableBucket(20001L, 0); + logTablet = + logManager.getOrCreateLog( + tempDir, physicalPath, tableBucket, LogFormat.ARROW, 1, true); + } + + @AfterEach + void tearDown() throws Exception { + if (kvManager != null) { + kvManager.shutdown(); + } + if (logManager != null) { + logManager.shutdown(); + } + if (localDiskManager != null) { + localDiskManager.close(); + } + } + + // ---- Helper methods ---- + + private KvTablet createSentinel() { + KvTablet tablet = + KvTablet.createLazySentinel( + physicalPath, + tableBucket, + logTablet, + TestingMetricGroups.TABLET_SERVER_METRICS); + kvManager.registerKv(tableBucket, tablet); + return tablet; + } + + private KvTabletLazyLifecycle configureLifecycle(KvTablet sentinel, ManualClock clock) { + KvTabletLazyLifecycle lifecycle = sentinel.getLifecycle(); + lifecycle.configureLazyOpen( + clock, + kvManager.getOpenSemaphore(), + kvManager.getOpenTimeoutMs(), + kvManager.getFailedBackoffBaseMs(), + kvManager.getFailedBackoffMaxMs(), + KvManager.RELEASE_DRAIN_TIMEOUT_MS); + lifecycle.setLeaderEpochSupplier(() -> 1); + lifecycle.setBucketEpochSupplier(() -> 1); + lifecycle.setTabletDirSupplier( + () -> + kvManager.getTabletDirPath( + logTablet.getDataDir(), physicalPath, tableBucket)); + lifecycle.setOpenCallback( + hasLocal -> + kvManager.createKvTabletUnregistered( + physicalPath, + tableBucket, + logTablet, + KvFormat.COMPACTED, + new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), + new TableConfig(new Configuration()), + DEFAULT_COMPRESSION, + null)); + lifecycle.setCommitCallback(kv -> {}); + lifecycle.setReleaseCallback(kv -> {}); + lifecycle.setDropCallback(kv -> {}); + return lifecycle; + } + + private void flushAndAwait(KvTablet tablet) throws Exception { + long offset = logTablet.localLogEndOffset(); + tablet.requestFlush(offset, NOPErrorHandler.INSTANCE); + CommonTestUtils.waitUntil( + () -> tablet.getFlushedLogOffset() >= offset, + Duration.ofSeconds(10), + "KV operation did not complete"); + } + + // ---- Tests ---- + + @Test + void testAcquireGuardTriggersOpen() throws Exception { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); + + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + // Verify we can use the tablet inside the guard + List values = + sentinel.multiGet(Collections.singletonList("k1".getBytes())); + assertThat(values).hasSize(1); + assertThat(values.get(0)).isNull(); + } + } + + @Test + void testAcquireGuardFastPath() throws Exception { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + // Track open callback invocations to prove fast path skips it + AtomicInteger openCount = new AtomicInteger(0); + lifecycle.setOpenCallback( + hasLocal -> { + openCount.incrementAndGet(); + return kvManager.createKvTabletUnregistered( + physicalPath, + tableBucket, + logTablet, + KvFormat.COMPACTED, + new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), + new TableConfig(new Configuration()), + DEFAULT_COMPRESSION, + null); + }); + + // First open via slow path + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + } + assertThat(openCount.get()).isEqualTo(1); + + // Second acquireGuard should take the fast path (already OPEN, no openCallback) + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + List values = + sentinel.multiGet(Collections.singletonList("nonexistent".getBytes())); + assertThat(values).hasSize(1); + assertThat(values.get(0)).isNull(); + } + // openCallback must NOT have been called again + assertThat(openCount.get()).isEqualTo(1); + } + + @Test + void testFailedBackoffRejectsOpen() { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + // Override openCallback to throw + lifecycle.setOpenCallback( + hasLocal -> { + throw new RuntimeException("simulated open failure"); + }); + + // First acquireGuard should fail + assertThatThrownBy(sentinel::acquireGuard).isInstanceOf(RuntimeException.class); + + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.FAILED); + + // Subsequent acquireGuard within cooldown should throw with cooldown message + assertThatThrownBy(sentinel::acquireGuard) + .isInstanceOf(KvStorageException.class) + .hasMessageContaining("cooldown"); + } + + @Test + void testFailedBackoffRetryAfterCooldown() throws Exception { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + // Override openCallback to throw + lifecycle.setOpenCallback( + hasLocal -> { + throw new RuntimeException("simulated open failure"); + }); + + // First acquireGuard fails + assertThatThrownBy(sentinel::acquireGuard).isInstanceOf(RuntimeException.class); + + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.FAILED); + + // Advance clock past the backoff (5000 * 2^1 = 10000ms for failureCount=1) + clock.advanceTime(11, TimeUnit.SECONDS); + + // Fix the openCallback + lifecycle.setOpenCallback( + hasLocal -> + kvManager.createKvTabletUnregistered( + physicalPath, + tableBucket, + logTablet, + KvFormat.COMPACTED, + new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), + new TableConfig(new Configuration()), + DEFAULT_COMPRESSION, + null)); + + // Now acquireGuard should succeed + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + } + } + + @Test + void testReleaseKvSuccess() throws Exception { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + // Open and write+flush data + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + KvRecord record = recordFactory.ofRecord("k1".getBytes(), new Object[] {1, "a"}); + KvRecordBatch batch = batchFactory.ofRecords(Collections.singletonList(record)); + sentinel.putAsLeader(batch, null); + flushAndAwait(sentinel); + } + + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + + // Capture flushed offset before release (detach may reset it) + long flushedOffsetBeforeRelease = sentinel.getFlushedLogOffset(); + + // Release should succeed + boolean released = sentinel.releaseKv(); + assertThat(released).isTrue(); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); + // Row count and flushed offset should be cached + assertThat(lifecycle.getCachedRowCount()).isEqualTo(1L); + assertThat(lifecycle.getCachedFlushedLogOffset()).isEqualTo(flushedOffsetBeforeRelease); + } + + @Test + void testReleaseKvRejectsUnflushedData() throws Exception { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + // Open and write data WITHOUT flush + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + KvRecord record = recordFactory.ofRecord("k1".getBytes(), new Object[] {1, "a"}); + KvRecordBatch batch = batchFactory.ofRecords(Collections.singletonList(record)); + sentinel.putAsLeader(batch, null); + // Intentionally NOT flushing + } + + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + + // Release should fail because data is unflushed + boolean released = sentinel.releaseKv(); + assertThat(released).isFalse(); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + } + + @Test + void testReleaseAndReopenPreservesData() throws Exception { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + // Open, write+flush + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + KvRecord record = recordFactory.ofRecord("k1".getBytes(), new Object[] {1, "a"}); + KvRecordBatch batch = batchFactory.ofRecords(Collections.singletonList(record)); + sentinel.putAsLeader(batch, null); + flushAndAwait(sentinel); + } + + // Release to LAZY + boolean released = sentinel.releaseKv(); + assertThat(released).isTrue(); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); + + // Reopen via acquireGuard and verify data is still readable + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + List values = + sentinel.multiGet(Collections.singletonList("k1".getBytes())); + assertThat(values).hasSize(1); + byte[] expected = + ValueEncoder.encodeValue( + SCHEMA_ID, compactedRow(BASE_ROW_TYPE, new Object[] {1, "a"})); + assertThat(values.get(0).toByteArray()).isEqualTo(expected); + } + } + + @Test + void testFlushRequestWhileReleaseDrainsWriter() throws Exception { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + FutureTask releaseTask = new FutureTask<>(sentinel::releaseKv); + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + new Thread(releaseTask, "release-during-flush-request").start(); + CommonTestUtils.waitUntil( + () -> lifecycle.getLazyState() == KvTabletLazyLifecycle.LazyState.RELEASING, + Duration.ofSeconds(5), + "Timed out waiting for RELEASING state"); + KvRecord record = recordFactory.ofRecord("k1".getBytes(), new Object[] {1, "a"}); + sentinel.putAsLeader(batchFactory.ofRecords(Collections.singletonList(record)), null); + flushAndAwait(sentinel); + assertThat(sentinel.getFlushedLogOffset()).isEqualTo(logTablet.localLogEndOffset()); + } + releaseTask.get(5, TimeUnit.SECONDS); + } + + @Test + void testReleaseKvRejectsWriteThatCompletesDuringDrain() throws Exception { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + KvTablet.Guard guard = sentinel.acquireGuard(); + FutureTask releaseTask = new FutureTask<>(sentinel::releaseKv); + Thread releaseThread = new Thread(releaseTask, "release-during-write"); + releaseThread.start(); + + CommonTestUtils.waitUntil( + () -> lifecycle.getLazyState() == KvTabletLazyLifecycle.LazyState.RELEASING, + Duration.ofSeconds(5), + "Timed out waiting for RELEASING state"); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.RELEASING); + + KvRecord record = recordFactory.ofRecord("k1".getBytes(), new Object[] {1, "a"}); + KvRecordBatch batch = batchFactory.ofRecords(Collections.singletonList(record)); + sentinel.putAsLeader(batch, null); + guard.close(); + assertThat(releaseTask.get(5, TimeUnit.SECONDS)).isFalse(); + + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + assertThat(sentinel.getKvPreWriteBuffer().pendingFlushBytes()).isPositive(); + } + + @Test + void testDropKvFromLazy() { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); + + sentinel.dropKvLazy(); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); + } + + @Test + void testDropKvFromOpen() throws Exception { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + // Open the tablet + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + } + + File kvDir = sentinel.getKvTabletDir(); + assertThat(kvDir).isNotNull(); + assertThat(kvDir).exists(); + + sentinel.dropKvLazy(); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); + assertThat(sentinel.getRocksDBKv()).isNull(); + assertThat(kvDir).doesNotExist(); + } + + @Test + void testDropKvDuringReleasingRollbackStillClosesAndDeletes() throws Exception { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + KvRecord record = recordFactory.ofRecord("k1".getBytes(), new Object[] {1, "a"}); + KvRecordBatch batch = batchFactory.ofRecords(Collections.singletonList(record)); + sentinel.putAsLeader(batch, null); + flushAndAwait(sentinel); + } + + File kvDir = sentinel.getKvTabletDir(); + assertThat(kvDir).isNotNull(); + assertThat(kvDir).exists(); + + CountDownLatch releaseCallbackEntered = new CountDownLatch(1); + CountDownLatch allowReleaseCallbackFinish = new CountDownLatch(1); + lifecycle.setReleaseCallback( + kv -> { + releaseCallbackEntered.countDown(); + try { + allowReleaseCallbackFinish.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + throw new RuntimeException("simulated release failure"); + }); + + FutureTask releaseTask = new FutureTask<>(sentinel::releaseKv); + Thread releaseThread = new Thread(releaseTask, "release-with-rollback"); + releaseThread.start(); + + assertThat(releaseCallbackEntered.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.RELEASING); + + FutureTask dropTask = + new FutureTask<>( + () -> { + sentinel.dropKvLazy(); + return null; + }); + Thread dropThread = new Thread(dropTask, "drop-during-release-rollback"); + dropThread.start(); + + allowReleaseCallbackFinish.countDown(); + assertThat(releaseTask.get(5, TimeUnit.SECONDS)).isFalse(); + dropTask.get(5, TimeUnit.SECONDS); + + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); + assertThat(sentinel.getRocksDBKv()).isNull(); + assertThat(kvDir).doesNotExist(); + } + + @Test + void testCommitCallbackFailureDoesNotFailOpen() throws Exception { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + lifecycle.setCommitCallback( + kv -> { + throw new RuntimeException("simulated commit callback failure"); + }); + + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + assertThat(sentinel.getRocksDBKv()).isNotNull(); + } + } + + @Test + void testDropWaitsForCommitCallbackCompletion() throws Exception { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + CountDownLatch commitCallbackEntered = new CountDownLatch(1); + CountDownLatch allowCommitCallbackFinish = new CountDownLatch(1); + lifecycle.setCommitCallback( + kv -> { + commitCallbackEntered.countDown(); + try { + allowCommitCallbackFinish.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + + AtomicReference openFailure = new AtomicReference<>(); + Thread openThread = + new Thread( + () -> { + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + // no-op + } catch (Throwable t) { + openFailure.set(t); + } + }, + "open-with-blocked-commit"); + openThread.start(); + + assertThat(commitCallbackEntered.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + + FutureTask dropTask = + new FutureTask<>( + () -> { + sentinel.dropKvLazy(); + return null; + }); + Thread dropThread = new Thread(dropTask, "drop-during-commit"); + dropThread.start(); + + // Wait for drop thread to block (commit callback holds the lock) + CommonTestUtils.waitUntil( + () -> { + Thread.State s = dropThread.getState(); + return s == Thread.State.WAITING || s == Thread.State.TIMED_WAITING; + }, + Duration.ofSeconds(5), + "Timed out waiting for drop thread to block"); + assertThat(dropTask.isDone()).isFalse(); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + + allowCommitCallbackFinish.countDown(); + openThread.join(TimeUnit.SECONDS.toMillis(5)); + assertThat(openThread.isAlive()).isFalse(); + dropTask.get(5, TimeUnit.SECONDS); + + // The open thread may or may not see a KvStorageException depending on the race + // between guard.close() and dropKvLazy() transitioning to CLOSED. Both outcomes are valid: + // - null: guard closed before drop detached RocksDB + // - KvStorageException: drop completed while open thread was still in acquireGuard + Throwable failure = openFailure.get(); + assertThat(failure == null || failure instanceof KvStorageException) + .as( + "Expected null or KvStorageException but got: %s", + failure != null ? failure.getClass().getName() : "null") + .isTrue(); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); + assertThat(sentinel.getRocksDBKv()).isNull(); + } + + @Test + void testCloseMarksLazyTabletClosed() throws Exception { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + } + + sentinel.close(); + + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); + assertThat(sentinel.getRocksDBKv()).isNull(); + assertThatThrownBy(sentinel::acquireGuard).isInstanceOf(KvStorageException.class); + } + + @Test + void testSnapshotExecutorDoesNotRunAgainstReopenedTablet() throws Exception { + KvTablet sentinel = createSentinel(); + configureLifecycle(sentinel, new ManualClock(System.currentTimeMillis())); + Executor executor; + try (KvTablet.Guard ignored = sentinel.acquireGuard()) { + executor = sentinel.getGuardedExecutor(); + } + assertThat(sentinel.releaseKv()).isTrue(); + try (KvTablet.Guard ignored = sentinel.acquireGuard()) { + AtomicInteger executions = new AtomicInteger(); + executor.execute(executions::incrementAndGet); + assertThat(executions).hasValue(0); + sentinel.getGuardedExecutor().execute(executions::incrementAndGet); + assertThat(executions).hasValue(1); + } + } + + @Test + void testGuardedExecutorPinsOpenTabletUntilTaskCompletes() throws Exception { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + } + + CountDownLatch taskStarted = new CountDownLatch(1); + CountDownLatch allowTaskComplete = new CountDownLatch(1); + FutureTask guardedTask = + new FutureTask<>( + () -> { + sentinel.getGuardedExecutor() + .execute( + () -> { + taskStarted.countDown(); + try { + allowTaskComplete.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + return null; + }); + Thread taskThread = new Thread(guardedTask, "guarded-executor-task"); + taskThread.start(); + + assertThat(taskStarted.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(sentinel.getActivePins()).isEqualTo(1); + + FutureTask releaseTask = new FutureTask<>(sentinel::releaseKv); + Thread releaseThread = new Thread(releaseTask, "release-task"); + releaseThread.start(); + + CommonTestUtils.waitUntil( + () -> lifecycle.getLazyState() == KvTabletLazyLifecycle.LazyState.RELEASING, + Duration.ofSeconds(5), + "Timed out waiting for RELEASING state"); + assertThat(releaseTask.isDone()).isFalse(); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.RELEASING); + + allowTaskComplete.countDown(); + assertThat(releaseTask.get(5, TimeUnit.SECONDS)).isTrue(); + taskThread.join(TimeUnit.SECONDS.toMillis(5)); + assertThat(taskThread.isAlive()).isFalse(); + + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); + assertThat(sentinel.getActivePins()).isZero(); + } + + @Test + void testGaugeUpdatesOnTransitions() throws Exception { + TabletServerMetricGroup mg = TestingMetricGroups.TABLET_SERVER_METRICS; + // Record baseline counts before this test + int baseLazy = mg.kvTabletLazyCount().get(); + int baseOpen = mg.kvTabletOpenCount().get(); + int baseFailed = mg.kvTabletFailedCount().get(); + + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + // After configureLazyOpen: LAZY +1 + assertThat(mg.kvTabletLazyCount().get() - baseLazy).isEqualTo(1); + assertThat(mg.kvTabletOpenCount().get() - baseOpen).isEqualTo(0); + assertThat(mg.kvTabletFailedCount().get() - baseFailed).isEqualTo(0); + + // Open: LAZY -1, OPEN +1 + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + KvRecord record = recordFactory.ofRecord("k1".getBytes(), new Object[] {1, "a"}); + KvRecordBatch batch = batchFactory.ofRecords(Collections.singletonList(record)); + sentinel.putAsLeader(batch, null); + flushAndAwait(sentinel); + } + assertThat(mg.kvTabletLazyCount().get() - baseLazy).isEqualTo(0); + assertThat(mg.kvTabletOpenCount().get() - baseOpen).isEqualTo(1); + + // Release: OPEN -1, LAZY +1 + boolean released = sentinel.releaseKv(); + assertThat(released).isTrue(); + assertThat(mg.kvTabletLazyCount().get() - baseLazy).isEqualTo(1); + assertThat(mg.kvTabletOpenCount().get() - baseOpen).isEqualTo(0); + + // Open again: LAZY -1, OPEN +1 + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + } + assertThat(mg.kvTabletLazyCount().get() - baseLazy).isEqualTo(0); + assertThat(mg.kvTabletOpenCount().get() - baseOpen).isEqualTo(1); + + // Drop: OPEN -1 + sentinel.dropKvLazy(); + assertThat(mg.kvTabletOpenCount().get() - baseOpen).isEqualTo(0); + assertThat(mg.kvTabletLazyCount().get() - baseLazy).isEqualTo(0); + assertThat(mg.kvTabletFailedCount().get() - baseFailed).isEqualTo(0); + } + + @Test + void testEpochFencingRejectsStaleOpenOnLeaderEpochChange() throws Exception { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + AtomicInteger leaderEpoch = new AtomicInteger(1); + lifecycle.setLeaderEpochSupplier(leaderEpoch::get); + + CountDownLatch openStarted = new CountDownLatch(1); + CountDownLatch allowOpenFinish = new CountDownLatch(1); + + lifecycle.setOpenCallback( + hasLocal -> { + openStarted.countDown(); + try { + allowOpenFinish.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return kvManager.createKvTabletUnregistered( + physicalPath, + tableBucket, + logTablet, + KvFormat.COMPACTED, + new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), + new TableConfig(new Configuration()), + DEFAULT_COMPRESSION, + null); + }); + + AtomicReference openFailure = new AtomicReference<>(); + Thread openThread = + new Thread( + () -> { + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + // no-op + } catch (Throwable t) { + openFailure.set(t); + } + }, + "epoch-fence-leader"); + openThread.start(); + + assertThat(openStarted.await(5, TimeUnit.SECONDS)).isTrue(); + + // Change leader epoch while open is in progress + leaderEpoch.set(2); + allowOpenFinish.countDown(); + + openThread.join(TimeUnit.SECONDS.toMillis(5)); + assertThat(openThread.isAlive()).isFalse(); + + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.FAILED); + assertThat(openFailure.get()).isNotNull(); + } + + @Test + void testEpochFencingRejectsStaleOpenOnBucketEpochChange() throws Exception { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + AtomicInteger bucketEpoch = new AtomicInteger(1); + lifecycle.setBucketEpochSupplier(bucketEpoch::get); + + CountDownLatch openStarted = new CountDownLatch(1); + CountDownLatch allowOpenFinish = new CountDownLatch(1); + + lifecycle.setOpenCallback( + hasLocal -> { + openStarted.countDown(); + try { + allowOpenFinish.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return kvManager.createKvTabletUnregistered( + physicalPath, + tableBucket, + logTablet, + KvFormat.COMPACTED, + new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), + new TableConfig(new Configuration()), + DEFAULT_COMPRESSION, + null); + }); + + AtomicReference openFailure = new AtomicReference<>(); + Thread openThread = + new Thread( + () -> { + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + // no-op + } catch (Throwable t) { + openFailure.set(t); + } + }, + "epoch-fence-bucket"); + openThread.start(); + + assertThat(openStarted.await(5, TimeUnit.SECONDS)).isTrue(); + + // Change bucket epoch while open is in progress + bucketEpoch.set(2); + allowOpenFinish.countDown(); + + openThread.join(TimeUnit.SECONDS.toMillis(5)); + assertThat(openThread.isAlive()).isFalse(); + + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.FAILED); + assertThat(openFailure.get()).isNotNull(); + } + + @Test + void testConcurrentAcquireGuardOnlyOpensOnce() throws Exception { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + AtomicInteger openCount = new AtomicInteger(0); + CountDownLatch openStarted = new CountDownLatch(1); + CountDownLatch allowOpenFinish = new CountDownLatch(1); + + lifecycle.setOpenCallback( + hasLocal -> { + openCount.incrementAndGet(); + openStarted.countDown(); + try { + allowOpenFinish.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return kvManager.createKvTabletUnregistered( + physicalPath, + tableBucket, + logTablet, + KvFormat.COMPACTED, + new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), + new TableConfig(new Configuration()), + DEFAULT_COMPRESSION, + null); + }); + + int threadCount = 5; + CountDownLatch barrier = new CountDownLatch(1); + AtomicInteger successCount = new AtomicInteger(0); + AtomicInteger failureCount = new AtomicInteger(0); + Thread[] threads = new Thread[threadCount]; + + for (int i = 0; i < threadCount; i++) { + threads[i] = + new Thread( + () -> { + try { + barrier.await(5, TimeUnit.SECONDS); + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + successCount.incrementAndGet(); + } + } catch (Exception e) { + failureCount.incrementAndGet(); + } + }, + "concurrent-acquire-" + i); + threads[i].start(); + } + + // Release all threads at once + barrier.countDown(); + + // Wait for the opener thread to enter doSlowOpen + assertThat(openStarted.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPENING); + + // Let the open complete + allowOpenFinish.countDown(); + + // Wait for all threads to finish + for (Thread t : threads) { + t.join(TimeUnit.SECONDS.toMillis(10)); + assertThat(t.isAlive()).isFalse(); + } + + assertThat(openCount.get()).isEqualTo(1); + assertThat(successCount.get()).isEqualTo(threadCount); + assertThat(failureCount.get()).isZero(); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + } + + @Test + void testDropKvDuringOpeningFencesAndCloses() throws Exception { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + CountDownLatch openStarted = new CountDownLatch(1); + CountDownLatch allowOpenFinish = new CountDownLatch(1); + + lifecycle.setOpenCallback( + hasLocal -> { + openStarted.countDown(); + try { + allowOpenFinish.await(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return kvManager.createKvTabletUnregistered( + physicalPath, + tableBucket, + logTablet, + KvFormat.COMPACTED, + new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), + new TableConfig(new Configuration()), + DEFAULT_COMPRESSION, + null); + }); + + AtomicReference openFailure = new AtomicReference<>(); + Thread openThread = + new Thread( + () -> { + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + // no-op + } catch (Throwable t) { + openFailure.set(t); + } + }, + "open-for-drop-fence"); + openThread.start(); + + assertThat(openStarted.await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPENING); + + FutureTask dropTask = + new FutureTask<>( + () -> { + sentinel.dropKvLazy(); + return null; + }); + Thread dropThread = new Thread(dropTask, "drop-during-opening"); + dropThread.start(); + + // Let the open finish — commitOpenResult should see generation mismatch + allowOpenFinish.countDown(); + + openThread.join(TimeUnit.SECONDS.toMillis(5)); + assertThat(openThread.isAlive()).isFalse(); + dropTask.get(5, TimeUnit.SECONDS); + + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); + assertThat(sentinel.getRocksDBKv()).isNull(); + assertThat(openFailure.get()).isNotNull(); + } + + @Test + void testReleaseDrainTimeoutRollsBackToOpen() throws Exception { + KvTablet sentinel = createSentinel(); + + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTabletLazyLifecycle lifecycle = sentinel.getLifecycle(); + lifecycle.configureLazyOpen( + clock, + kvManager.getOpenSemaphore(), + kvManager.getOpenTimeoutMs(), + kvManager.getFailedBackoffBaseMs(), + kvManager.getFailedBackoffMaxMs(), + 100L); // 100ms drain timeout + lifecycle.setLeaderEpochSupplier(() -> 1); + lifecycle.setBucketEpochSupplier(() -> 1); + lifecycle.setTabletDirSupplier( + () -> + kvManager.getTabletDirPath( + logTablet.getDataDir(), physicalPath, tableBucket)); + lifecycle.setOpenCallback( + hasLocal -> + kvManager.createKvTabletUnregistered( + physicalPath, + tableBucket, + logTablet, + KvFormat.COMPACTED, + new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), + new TableConfig(new Configuration()), + DEFAULT_COMPRESSION, + null)); + lifecycle.setCommitCallback(kv -> {}); + lifecycle.setReleaseCallback(kv -> {}); + lifecycle.setDropCallback(kv -> {}); + + // Acquire a guard and hold it to prevent drain from completing + KvTablet.Guard guard = sentinel.acquireGuard(); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + + // Start releaseKv in a background thread — it will block waiting for drain + FutureTask releaseTask = new FutureTask<>(sentinel::releaseKv); + Thread releaseThread = new Thread(releaseTask, "release-drain-timeout"); + releaseThread.start(); + + // Wait for the release thread to enter RELEASING state + CommonTestUtils.waitUntil( + () -> lifecycle.getLazyState() == KvTabletLazyLifecycle.LazyState.RELEASING, + Duration.ofSeconds(5), + "Timed out waiting for RELEASING state"); + + // Advance clock past drain timeout to trigger rollback deterministically + clock.advanceTime(200, TimeUnit.MILLISECONDS); + + // The release should timeout and roll back to OPEN + boolean released = releaseTask.get(5, TimeUnit.SECONDS); + assertThat(released).isFalse(); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + + // Close the guard and verify pins are drained + guard.close(); + assertThat(sentinel.getActivePins()).isZero(); + } + + @Test + void testReleaseCallbackFailureRollsBackToOpenAndTabletRemainsFunctional() throws Exception { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + // Open and write+flush data + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + KvRecord record = recordFactory.ofRecord("k1".getBytes(), new Object[] {1, "a"}); + KvRecordBatch batch = batchFactory.ofRecords(Collections.singletonList(record)); + sentinel.putAsLeader(batch, null); + flushAndAwait(sentinel); + } + + // Set release callback to throw + lifecycle.setReleaseCallback( + kv -> { + throw new RuntimeException("simulated release callback failure"); + }); + + // releaseKv should return false due to callback failure + boolean released = sentinel.releaseKv(); + assertThat(released).isFalse(); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + + // Verify tablet is still functional — acquireGuard and multiGet should work + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + List values = + sentinel.multiGet(Collections.singletonList("k1".getBytes())); + assertThat(values).hasSize(1); + byte[] expected = + ValueEncoder.encodeValue( + SCHEMA_ID, compactedRow(BASE_ROW_TYPE, new Object[] {1, "a"})); + assertThat(values.get(0).toByteArray()).isEqualTo(expected); + } + + // Fix the callback and verify subsequent release succeeds + lifecycle.setReleaseCallback(kv -> {}); + released = sentinel.releaseKv(); + assertThat(released).isTrue(); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); + } + + @Test + void testDropKvFromFailed() { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + // Override openCallback to throw + lifecycle.setOpenCallback( + hasLocal -> { + throw new RuntimeException("simulated open failure"); + }); + + // First acquireGuard should fail + assertThatThrownBy(sentinel::acquireGuard).isInstanceOf(RuntimeException.class); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.FAILED); + + // Drop from FAILED state + sentinel.dropKvLazy(); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); + } + + @Test + void testExponentialBackoffEscalatesAcrossMultipleFailures() throws Exception { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + // Override openCallback to throw + lifecycle.setOpenCallback( + hasLocal -> { + throw new RuntimeException("simulated open failure"); + }); + + // Failure 1: failureCount becomes 1, cooldown = 5000 * 2^1 = 10000ms + assertThatThrownBy(sentinel::acquireGuard).isInstanceOf(RuntimeException.class); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.FAILED); + + // Advance 9s — still within 10s cooldown + clock.advanceTime(9, TimeUnit.SECONDS); + assertThatThrownBy(sentinel::acquireGuard) + .isInstanceOf(KvStorageException.class) + .hasMessageContaining("cooldown"); + + // Advance 2s more (total 11s, past 10s cooldown) — retry and fail again + clock.advanceTime(2, TimeUnit.SECONDS); + assertThatThrownBy(sentinel::acquireGuard).isInstanceOf(RuntimeException.class); + + // Failure 2: failureCount becomes 2, cooldown = 5000 * 2^2 = 20000ms + // Advance 19s — still within 20s cooldown + clock.advanceTime(19, TimeUnit.SECONDS); + assertThatThrownBy(sentinel::acquireGuard) + .isInstanceOf(KvStorageException.class) + .hasMessageContaining("cooldown"); + + // Advance 2s more (total 21s, past 20s cooldown) — retry and fail again + clock.advanceTime(2, TimeUnit.SECONDS); + assertThatThrownBy(sentinel::acquireGuard).isInstanceOf(RuntimeException.class); + + // Failure 3: failureCount becomes 3, cooldown = 5000 * 2^3 = 40000ms + // Advance 39s — still within 40s cooldown + clock.advanceTime(39, TimeUnit.SECONDS); + assertThatThrownBy(sentinel::acquireGuard) + .isInstanceOf(KvStorageException.class) + .hasMessageContaining("cooldown"); + + // Advance 2s more (total 41s, past 40s cooldown) — fix callback and verify recovery + clock.advanceTime(2, TimeUnit.SECONDS); + lifecycle.setOpenCallback( + hasLocal -> + kvManager.createKvTabletUnregistered( + physicalPath, + tableBucket, + logTablet, + KvFormat.COMPACTED, + new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), + new TableConfig(new Configuration()), + DEFAULT_COMPRESSION, + null)); + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + } + } + + @Test + void testGuardDoubleCloseDoesNotDecrementPinsBelowZero() { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + configureLifecycle(sentinel, clock); + + KvTablet.Guard guard = sentinel.acquireGuard(); + assertThat(sentinel.getActivePins()).isEqualTo(1); + + guard.close(); + assertThat(sentinel.getActivePins()).isZero(); + + // Second close should be idempotent — pins must not go negative + guard.close(); + assertThat(sentinel.getActivePins()).isZero(); + } + + @Test + void testAcquireGuardOnClosedTabletThrows() { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + configureLifecycle(sentinel, clock); + + // Open the tablet + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + assertThat(sentinel.getLifecycle().getLazyState()) + .isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + } + + // Drop transitions to CLOSED + sentinel.dropKvLazy(); + assertThat(sentinel.getLifecycle().getLazyState()) + .isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); + + // acquireGuard on a CLOSED tablet must throw + assertThatThrownBy(sentinel::acquireGuard) + .isInstanceOf(KvStorageException.class) + .hasMessageContaining("closed"); + } + + @Test + void testGaugeUpdatesOnFailedTransition() { + TabletServerMetricGroup mg = TestingMetricGroups.TABLET_SERVER_METRICS; + int baseLazy = mg.kvTabletLazyCount().get(); + int baseOpen = mg.kvTabletOpenCount().get(); + int baseFailed = mg.kvTabletFailedCount().get(); + + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + + // After configureLazyOpen: LAZY +1 + assertThat(mg.kvTabletLazyCount().get() - baseLazy).isEqualTo(1); + + // Override openCallback to throw + lifecycle.setOpenCallback( + hasLocal -> { + throw new RuntimeException("simulated open failure"); + }); + + // acquireGuard should fail and transition to FAILED + assertThatThrownBy(sentinel::acquireGuard).isInstanceOf(RuntimeException.class); + + // LAZY back to baseline, FAILED +1, OPEN unchanged + assertThat(mg.kvTabletLazyCount().get() - baseLazy).isEqualTo(0); + assertThat(mg.kvTabletFailedCount().get() - baseFailed).isEqualTo(1); + assertThat(mg.kvTabletOpenCount().get() - baseOpen).isEqualTo(0); + + // Drop from FAILED state + sentinel.dropKvLazy(); + + // All gauges back to baseline + assertThat(mg.kvTabletLazyCount().get() - baseLazy).isEqualTo(0); + assertThat(mg.kvTabletOpenCount().get() - baseOpen).isEqualTo(0); + assertThat(mg.kvTabletFailedCount().get() - baseFailed).isEqualTo(0); + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaLazyOpenTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaLazyOpenTest.java new file mode 100644 index 00000000000..79e991c622f --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaLazyOpenTest.java @@ -0,0 +1,410 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.replica; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.fs.FsPath; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.record.ChangeType; +import org.apache.fluss.record.DefaultValueRecordBatch; +import org.apache.fluss.record.KvRecord; +import org.apache.fluss.record.KvRecordBatch; +import org.apache.fluss.record.KvRecordTestUtils; +import org.apache.fluss.row.encode.CompactedKeyEncoder; +import org.apache.fluss.row.encode.ValueEncoder; +import org.apache.fluss.rpc.protocol.MergeMode; +import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; +import org.apache.fluss.server.kv.KvTablet; +import org.apache.fluss.server.kv.scan.ScannerContext; +import org.apache.fluss.server.kv.snapshot.CompletedSnapshot; +import org.apache.fluss.server.kv.snapshot.KvFileHandle; +import org.apache.fluss.server.kv.snapshot.KvFileHandleAndLocalPath; +import org.apache.fluss.server.kv.snapshot.KvSnapshotHandle; +import org.apache.fluss.server.log.LogAppendInfo; +import org.apache.fluss.server.zk.NOPErrorHandler; +import org.apache.fluss.server.zk.data.LeaderAndIsr; +import org.apache.fluss.testutils.common.CommonTestUtils; +import org.apache.fluss.utils.ByteArraySlice; + +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.apache.fluss.compression.ArrowCompressionInfo.DEFAULT_COMPRESSION; +import static org.apache.fluss.record.LogRecordBatch.CURRENT_LOG_MAGIC_VALUE; +import static org.apache.fluss.record.LogRecordBatchFormat.NO_BATCH_SEQUENCE; +import static org.apache.fluss.record.LogRecordBatchFormat.NO_WRITER_ID; +import static org.apache.fluss.record.TestData.DATA1_PHYSICAL_TABLE_PATH_PK; +import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE; +import static org.apache.fluss.record.TestData.DATA1_TABLE_ID_PK; +import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID; +import static org.apache.fluss.testutils.DataTestUtils.compactedRow; +import static org.apache.fluss.testutils.DataTestUtils.createBasicMemoryLogRecords; +import static org.apache.fluss.utils.FileUtils.deleteDirectory; +import static org.apache.fluss.utils.Preconditions.checkNotNull; +import static org.assertj.core.api.Assertions.assertThat; + +/** Integration tests for lazy-open behavior in {@link Replica}. */ +final class ReplicaLazyOpenTest extends ReplicaTestBase { + + @Override + protected Configuration getServerConf() { + Configuration conf = super.getServerConf(); + conf.set(ConfigOptions.KV_LAZY_OPEN_ENABLED, true); + conf.set(ConfigOptions.KV_LAZY_OPEN_IDLE_TIMEOUT, Duration.ofHours(1)); + return conf; + } + + @Test + void testFirstLookupFlushesRecoveredRecordsBeyondHighWatermark() throws Exception { + Replica replica = + makeKvReplica(DATA1_PHYSICAL_TABLE_PATH_PK, new TableBucket(DATA1_TABLE_ID_PK, 1)); + replica.appendRecordsToFollower( + createBasicMemoryLogRecords( + DATA1_ROW_TYPE, + DEFAULT_SCHEMA_ID, + 0L, + -1L, + CURRENT_LOG_MAGIC_VALUE, + NO_WRITER_ID, + NO_BATCH_SEQUENCE, + Collections.singletonList(ChangeType.INSERT), + Collections.singletonList(new Object[] {1, "a"}), + LogFormat.ARROW, + DEFAULT_COMPRESSION)); + makeLeaderReplica(replica); + assertThat(replica.getLogHighWatermark()).isZero(); + assertThat(replica.getLocalLogEndOffset()).isEqualTo(1L); + + replica.lookups(Collections.singletonList(key(1))); + + CommonTestUtils.waitUntil( + () -> replica.getLogHighWatermark() == 1L, + Duration.ofSeconds(10), + "Recovered WAL did not become visible after lazy open"); + assertThat(replica.lookups(Collections.singletonList(key(1)))) + .extracting(ByteArraySlice::toByteArray) + .containsExactly(valueBytes(1, "a")); + } + + @Test + void testPutRecordsToLeaderTriggersLazyOpen() throws Exception { + Replica replica = makeLazyLeaderReplica(); + KvTablet kvTablet = checkNotNull(replica.getKvTablet()); + assertThat(kvTablet.isLazyOpen()).isFalse(); + assertThat(kvTablet.getRocksDBKv()).isNull(); + + LogAppendInfo appendInfo = putRecordsToLeaderAndFlush(replica, records(1, "a")); + + assertThat(appendInfo.lastOffset()).isEqualTo(0L); + CommonTestUtils.waitUntil( + () -> replica.getLogTablet().getHighWatermark() == appendInfo.lastOffset() + 1, + Duration.ofSeconds(10), + "KV operation did not complete"); + assertThat(kvTablet.isLazyOpen()).isTrue(); + assertThat(kvTablet.getRocksDBKv()).isNotNull(); + assertThat(replica.lookups(Collections.singletonList(key(1)))) + .extracting(ByteArraySlice::toByteArray) + .containsExactly(valueBytes(1, "a")); + } + + @Test + void testLookupsReopenReleasedLazyTablet() throws Exception { + Replica replica = makeLazyLeaderReplica(); + putRecordsToLeaderAndFlush(replica, records(1, "a", 2, "b")); + releaseToLazy(replica); + + List values = replica.lookups(Arrays.asList(key(1), key(2))); + + assertThat(values) + .extracting(ByteArraySlice::toByteArray) + .containsExactly(valueBytes(1, "a"), valueBytes(2, "b")); + assertThat(checkNotNull(replica.getKvTablet()).isLazyOpen()).isTrue(); + } + + @Test + void testPrefixLookupReopensReleasedLazyTablet() throws Exception { + Replica replica = makeLazyLeaderReplica(); + putRecordsToLeaderAndFlush(replica, records(1, "a", 2, "b")); + releaseToLazy(replica); + + List values = replica.prefixLookup(new byte[0]); + + assertThat(values) + .extracting(ByteArraySlice::toByteArray) + .containsExactly(valueBytes(1, "a"), valueBytes(2, "b")); + assertThat(checkNotNull(replica.getKvTablet()).isLazyOpen()).isTrue(); + } + + @Test + void testLimitScanReopensReleasedLazyTablet() throws Exception { + Replica replica = makeLazyLeaderReplica(); + putRecordsToLeaderAndFlush(replica, records(1, "a", 2, "b")); + releaseToLazy(replica); + + DefaultValueRecordBatch.Builder builder = DefaultValueRecordBatch.builder(); + builder.append(DEFAULT_SCHEMA_ID, compactedRow(DATA1_ROW_TYPE, new Object[] {1, "a"})); + builder.append(DEFAULT_SCHEMA_ID, compactedRow(DATA1_ROW_TYPE, new Object[] {2, "b"})); + + DefaultValueRecordBatch values = replica.limitKvScan(2); + + assertThat(values).isEqualTo(builder.build()); + assertThat(checkNotNull(replica.getKvTablet()).isLazyOpen()).isTrue(); + } + + @Test + void testScannerPreventsIdleReleaseUntilClosed() throws Exception { + Replica replica = makeLazyLeaderReplica(); + putRecordsToLeaderAndFlush(replica, records(1, "a")); + releaseToLazy(replica); + + ScannerContext scanner = + checkNotNull(scannerManager.createScanner(replica, null).getContext()); + KvTablet tablet = checkNotNull(replica.getKvTablet()); + try { + assertThat(tablet.isLazyOpen()).isTrue(); + assertThat(tablet.releaseKv()).isFalse(); + assertThat(scanner.isValid()).isTrue(); + assertThat(scanner.currentValue()).isEqualTo(valueBytes(1, "a")); + } finally { + scannerManager.closeScannersForBucket(replica.getTableBucket()); + } + releaseToLazy(replica); + assertThat(replica.lookups(Collections.singletonList(key(1)))) + .extracting(ByteArraySlice::toByteArray) + .containsExactly(valueBytes(1, "a")); + } + + @Test + void testGetRowCountReturnsCachedValueWithoutReopening() throws Exception { + Replica replica = makeLazyLeaderReplica(); + putRecordsToLeaderAndFlush(replica, records(1, "a", 2, "b")); + releaseToLazy(replica); + + long rowCount = replica.getRowCount(); + + assertThat(rowCount).isEqualTo(2L); + assertThat(checkNotNull(replica.getKvTablet()).isLazyOpen()).isFalse(); + } + + @Test + void testLatestKvSnapshotSizeFallsBackToCommittedSnapshotWhenReleased() throws Exception { + TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID_PK, 1); + Replica replica = makeLazyLeaderReplicaWithSharedSnapshotStore(tableBucket); + putRecordsToLeaderAndFlush(replica, records(1, "a", 2, "b")); + + // Arbitrary non-zero size to verify fallback to committed snapshot + long expectedSnapshotSize = 321L; + KvSnapshotHandle snapshotHandle = + KvSnapshotHandle.create( + Collections.singletonList( + KvFileHandleAndLocalPath.of( + new KvFileHandle( + new FsPath( + tempDir.getAbsolutePath(), + "snapshot-1.sst") + .toString(), + expectedSnapshotSize), + "snapshot-1.sst")), + Collections.emptyList(), + expectedSnapshotSize); + CompletedSnapshot completedSnapshot = + new CompletedSnapshot( + tableBucket, + 1L, + new FsPath(tempDir.getAbsolutePath(), "snapshot-1"), + snapshotHandle, + replica.getLocalLogEndOffset(), + 2L, + null); + snapshotReporter.commitKvSnapshot(completedSnapshot, 0, 0); + + releaseToLazy(replica); + + assertThat(replica.getKvSnapshotManager()).isNull(); + assertThat(replica.getLatestKvSnapshotSize()) + .isEqualTo(completedSnapshot.getSnapshotSize()); + assertThat(replica.logicalStorageKvSize()).isEqualTo(completedSnapshot.getSnapshotSize()); + } + + @Test + void testMakeFollowerDropsLazyTablet() throws Exception { + Replica replica = makeLazyLeaderReplica(); + KvTablet kvTablet = checkNotNull(replica.getKvTablet()); + assertThat(kvTablet.isLazyOpen()).isFalse(); + assertThat(kvTablet.getRocksDBKv()).isNull(); + + replica.makeFollower( + notifyLeaderAndIsr( + new TableBucket(DATA1_TABLE_ID_PK, 1), + Collections.singletonList(TABLET_SERVER_ID), + Collections.singletonList(TABLET_SERVER_ID), + 1)); + + assertThat(replica.getKvTablet()).isNull(); + } + + @Test + void testMakeFollowerDropsOpenLazyTablet() throws Exception { + Replica replica = makeLazyLeaderReplica(); + putRecordsToLeaderAndFlush(replica, records(1, "a")); + KvTablet kvTablet = checkNotNull(replica.getKvTablet()); + assertThat(kvTablet.isLazyOpen()).isTrue(); + + replica.makeFollower( + notifyLeaderAndIsr( + new TableBucket(DATA1_TABLE_ID_PK, 1), + Collections.singletonList(TABLET_SERVER_ID), + Collections.singletonList(TABLET_SERVER_ID), + 1)); + + assertThat(replica.getKvTablet()).isNull(); + } + + @Test + void testReopenFallsBackToFullInitWhenLocalDataCorrupted() throws Exception { + Replica replica = makeLazyLeaderReplica(); + putRecordsToLeaderAndFlush(replica, records(1, "a", 2, "b")); + + // Get the tablet directory path before release + KvTablet kvTablet = checkNotNull(replica.getKvTablet()); + File tabletDir = + kvManager.getTabletDirPath( + replica.getLogTablet().getDataDir(), + DATA1_PHYSICAL_TABLE_PATH_PK, + new TableBucket(DATA1_TABLE_ID_PK, 1)); + + releaseToLazy(replica); + + // Corrupt the local directory by deleting it + assertThat(tabletDir).exists(); + deleteDirectory(tabletDir); + assertThat(tabletDir).doesNotExist(); + + // Trigger reopen — should fall back to full init (no snapshot = fresh tablet) + List values = replica.lookups(Collections.singletonList(key(1))); + + // Tablet should be open and functional after fallback + assertThat(checkNotNull(replica.getKvTablet()).isLazyOpen()).isTrue(); + assertThat(values.get(0).toByteArray()).isEqualTo(valueBytes(1, "a")); + + // Verify the tablet is fully functional — can write new data + putRecordsToLeaderAndFlush(replica, records(3, "c")); + assertThat(replica.lookups(Collections.singletonList(key(3)))) + .extracting(ByteArraySlice::toByteArray) + .containsExactly(valueBytes(3, "c")); + } + + private Replica makeLazyLeaderReplica() throws Exception { + Replica replica = + makeKvReplica(DATA1_PHYSICAL_TABLE_PATH_PK, new TableBucket(DATA1_TABLE_ID_PK, 1)); + makeLeaderReplica(replica); + return replica; + } + + private Replica makeLazyLeaderReplicaWithSharedSnapshotStore(TableBucket tableBucket) + throws Exception { + Replica replica = + makeKvReplica( + DATA1_PHYSICAL_TABLE_PATH_PK, + tableBucket, + new TestSnapshotContext( + conf.getString(ConfigOptions.REMOTE_DATA_DIR), snapshotReporter)); + makeLeaderReplica(replica); + return replica; + } + + private void makeLeaderReplica(Replica replica) throws Exception { + replica.makeLeader( + notifyLeaderAndIsr( + new TableBucket(DATA1_TABLE_ID_PK, 1), + Collections.singletonList(TABLET_SERVER_ID), + Collections.singletonList(TABLET_SERVER_ID), + 0)); + } + + private NotifyLeaderAndIsrData notifyLeaderAndIsr( + TableBucket tableBucket, List replicas, List isr, int leaderEpoch) { + return new NotifyLeaderAndIsrData( + DATA1_PHYSICAL_TABLE_PATH_PK, + tableBucket, + replicas, + new LeaderAndIsr( + TABLET_SERVER_ID, + leaderEpoch, + isr, + Collections.emptyList(), + org.apache.fluss.server.coordinator.CoordinatorContext + .INITIAL_COORDINATOR_EPOCH, + leaderEpoch)); + } + + private LogAppendInfo putRecordsToLeaderAndFlush(Replica replica, KvRecordBatch kvRecords) + throws Exception { + LogAppendInfo appendInfo = + replica.putRecordsToLeader(kvRecords, null, MergeMode.DEFAULT, 0); + KvTablet tablet = checkNotNull(replica.getKvTablet()); + tablet.requestFlush(replica.getLocalLogEndOffset(), NOPErrorHandler.INSTANCE); + CommonTestUtils.waitUntil( + () -> + tablet.getFlushedLogOffset() >= replica.getLocalLogEndOffset() + && replica.getLogHighWatermark() >= replica.getLocalLogEndOffset(), + Duration.ofSeconds(10), + "KV operation did not complete"); + return appendInfo; + } + + private void releaseToLazy(Replica replica) throws Exception { + KvTablet kvTablet = checkNotNull(replica.getKvTablet()); + CommonTestUtils.waitUntil( + kvTablet::releaseKv, Duration.ofSeconds(10), "KV operation did not complete"); + assertThat(kvTablet.isLazyOpen()).isFalse(); + assertThat(kvTablet.getRocksDBKv()).isNull(); + } + + private KvRecordBatch records(Object... values) throws Exception { + KvRecordTestUtils.KvRecordFactory recordFactory = + KvRecordTestUtils.KvRecordFactory.of(DATA1_ROW_TYPE); + KvRecordTestUtils.KvRecordBatchFactory batchFactory = + KvRecordTestUtils.KvRecordBatchFactory.of(DEFAULT_SCHEMA_ID); + + KvRecord[] records = new KvRecord[values.length / 2]; + for (int i = 0; i < values.length; i += 2) { + records[i / 2] = + recordFactory.ofRecord( + key((Integer) values[i]), new Object[] {values[i], values[i + 1]}); + } + return batchFactory.ofRecords(records); + } + + private byte[] key(int id) { + return new CompactedKeyEncoder(DATA1_ROW_TYPE, new int[] {0}) + .encodeKey(compactedRow(DATA1_ROW_TYPE, new Object[] {id, ""})); + } + + private byte[] valueBytes(int id, String value) { + return ValueEncoder.encodeValue( + DEFAULT_SCHEMA_ID, compactedRow(DATA1_ROW_TYPE, new Object[] {id, value})); + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java index 50a9dc79ec2..1956c2bbab7 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java @@ -47,11 +47,13 @@ import org.apache.fluss.server.kv.snapshot.CompletedSnapshot; import org.apache.fluss.server.kv.snapshot.KvSnapshotDataDownloader; import org.apache.fluss.server.kv.snapshot.KvSnapshotDownloadSpec; +import org.apache.fluss.server.kv.snapshot.KvSnapshotHandle; import org.apache.fluss.server.kv.snapshot.TestingCompletedKvSnapshotCommitter; import org.apache.fluss.server.log.FetchParams; import org.apache.fluss.server.log.ListOffsetsParam; import org.apache.fluss.server.log.LogAppendInfo; import org.apache.fluss.server.log.LogReadInfo; +import org.apache.fluss.server.log.LogTablet; import org.apache.fluss.server.testutils.KvTestUtils; import org.apache.fluss.server.zk.data.LeaderAndIsr; import org.apache.fluss.testutils.DataTestUtils; @@ -1276,6 +1278,84 @@ private void verifyGetKeyValues( .containsExactlyElementsOf(expectValues); } + @Test + void testGetRowCountPkTableWithNullKvTablet() throws Exception { + // Create a PK table replica but do NOT make it leader, so kvTablet stays null. + TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID_PK, 1); + Replica kvReplica = makeKvReplica(DATA1_PHYSICAL_TABLE_PATH_PK, tableBucket); + + // Verify preconditions: this is a PK table and kvTablet is null. + assertThat(kvReplica.isKvTable()).isTrue(); + assertThat(kvReplica.getKvTablet()).isNull(); + + // Append some log records so logTablet has a non-zero row count. + LogTablet logTablet = kvReplica.getLogTablet(); + MemoryLogRecords records = genMemoryLogRecordsByObject(DATA1); + logTablet.appendAsLeader(records); + logTablet.updateHighWatermark(logTablet.localLogEndOffset()); + assertThat(logTablet.getRowCount()).isGreaterThan(0); + + // getRowCount() should return 0 for a PK table when kvTablet is null, + // NOT the logTablet row count. + assertThat(kvReplica.getRowCount()).isEqualTo(0L); + } + + @Test + void testCreateKvRollbackOnAllRetriesFailed() throws Exception { + // Create a SnapshotContext whose getSnapshotDataDownloader always throws. + // This forces initKvTablet() to fail on every retry attempt because the + // snapshot download will always fail. + TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID_PK, 1); + TestSnapshotContext failingSnapshotContext = + new TestSnapshotContext(conf.getString(ConfigOptions.REMOTE_DATA_DIR)) { + @Override + public FunctionWithException + getLatestCompletedSnapshotProvider() { + // Return a provider that always returns a fake snapshot, + // so initKvTablet takes the snapshot-restore path + return tb -> + new CompletedSnapshot( + tb, + 1L, + new FsPath("file:///non-existent-path/snapshot-1"), + KvSnapshotHandle.create( + Collections.emptyList(), + Collections.emptyList(), + 0), + 0L, + null, + null); + } + + @Override + public KvSnapshotDataDownloader getSnapshotDataDownloader() { + // Return a downloader that always throws to simulate download failure + return new KvSnapshotDataDownloader( + java.util.concurrent.Executors.newSingleThreadExecutor()) { + @Override + public void transferAllDataToDirectory( + KvSnapshotDownloadSpec kvSnapshotDownloadSpec, + org.apache.fluss.utils.CloseableRegistry closeableRegistry) + throws Exception { + throw new IOException("Simulated snapshot download failure"); + } + }; + } + }; + + Replica kvReplica = + makeKvReplica(DATA1_PHYSICAL_TABLE_PATH_PK, tableBucket, failingSnapshotContext); + + assertThatThrownBy(() -> makeKvReplicaAsLeader(kvReplica)) + .isInstanceOf(org.apache.fluss.exception.KvStorageException.class); + + // kvTablet should be null since all retries failed + assertThat(kvReplica.getKvTablet()).isNull(); + + // KvManager should not have a half-initialized entry for this bucket + assertThat(kvManager.getKv(tableBucket)).isEmpty(); + } + /** A scheduledExecutorService that will execute the scheduled task immediately. */ private static class ImmediateTriggeredScheduledExecutorService extends ManuallyTriggeredScheduledExecutorService { diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java index 73d41737d8b..12933e36fbe 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java @@ -62,6 +62,7 @@ import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; import org.apache.fluss.server.entity.NotifyLeaderAndIsrResultForBucket; import org.apache.fluss.server.entity.PutKvDataForBucket; +import org.apache.fluss.server.kv.KvManager; import org.apache.fluss.server.kv.KvStateLookupResult; import org.apache.fluss.server.kv.KvTablet; import org.apache.fluss.server.kv.historical.HistoricalKvKeyEncoder; @@ -75,12 +76,14 @@ import org.apache.fluss.server.metadata.PartitionMetadata; import org.apache.fluss.server.metadata.ServerInfo; import org.apache.fluss.server.metadata.TableMetadata; +import org.apache.fluss.server.metrics.group.TestingMetricGroups; import org.apache.fluss.server.replica.Replica; import org.apache.fluss.server.replica.ReplicaTestBase; import org.apache.fluss.server.zk.data.LeaderAndIsr; import org.apache.fluss.server.zk.data.TableRegistration; import org.apache.fluss.server.zk.data.lake.LakeTableHelper; import org.apache.fluss.server.zk.data.lake.LakeTableSnapshot; +import org.apache.fluss.testutils.common.CommonTestUtils; import org.apache.fluss.testutils.common.ManuallyTriggeredScheduledExecutorService; import org.apache.fluss.types.DataField; import org.apache.fluss.types.DataTypes; @@ -142,6 +145,73 @@ class HistoricalPartitionManagerTest extends ReplicaTestBase { new DataField("id", DataTypes.INT()), new DataField("region", DataTypes.STRING())); + @Test + void testHistoricalLazyOpenReleaseAndRecover() throws Exception { + replicaManager.shutdown(); + kvManager.shutdown(); + conf.set(ConfigOptions.KV_LAZY_OPEN_ENABLED, true); + kvManager = + KvManager.create( + conf, + zkClient, + logManager, + TestingMetricGroups.TABLET_SERVER_METRICS, + localDiskManager, + manualClock); + kvManager.startup(); + replicaManager = buildReplicaManager(testCoordinatorGateway); + replicaManager.startup(); + + TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); + Replica replica = replicaManager.getReplicaOrException(TABLE_BUCKET); + KvTablet tablet = replica.getKvTablet(); + assertThat(tablet).isNotNull(); + assertThat(tablet.isLazyOpen()).isFalse(); + TestingHistoricalLakeLookupManager lakeLookupManager = + new TestingHistoricalLakeLookupManager(lookupConfiguration()); + try (HistoricalPartitionManager manager = + createHistoricalPartitionManager(lakeLookupManager)) { + writeBatch( + manager, + replica, + ORIGINAL_PARTITION, + batch(tableInfo.getRowType(), upsert(1, "us", ORIGINAL_PARTITION, "v1"))); + flushAndWait(tablet, replica.getLocalLogEndOffset()); + assertThat(tablet.isLazyOpen()).isTrue(); + assertThat(replica.getKvSnapshotManager()).isNull(); + CommonTestUtils.waitUntil( + tablet::releaseKv, Duration.ofSeconds(10), "KV release did not finish"); + assertThat(tablet.getRocksDBKv()).isNull(); + replica.tryUpdateHistoricalCleanupOffset(replica.getLocalLogEndOffset(), () -> {}); + + assertThat( + replica.lookupHistoricalLocal( + ORIGINAL_PARTITION, Collections.singletonList(key(1, "us")))) + .hasSize(1); + assertHistoricalValue( + tablet, + ORIGINAL_PARTITION, + key(1, "us"), + tableInfo, + row(1, "us", ORIGINAL_PARTITION, "v1")); + assertThat(replica.getKvSnapshotManager()).isNull(); + assertThat(tablet.getHistoricalCleanupOffset()) + .isEqualTo(replica.getLocalLogEndOffset()); + writeBatch( + manager, + replica, + ORIGINAL_PARTITION, + batch(tableInfo.getRowType(), upsert(1, "us", ORIGINAL_PARTITION, "v2"))); + flushAndWait(tablet, replica.getLocalLogEndOffset()); + assertHistoricalValue( + tablet, + ORIGINAL_PARTITION, + key(1, "us"), + tableInfo, + row(1, "us", ORIGINAL_PARTITION, "v2")); + } + } + @Test void testResolvesMultipleLakeMissesWithoutPrewriteRollback() throws Exception { TableInfo tableInfo = registerHistoricalTableAndBecomeLeader(); @@ -1161,7 +1231,7 @@ TABLE_BUCKET, new FetchReqInfo(TABLE_ID, fetchOffset, Integer.MAX_VALUE)), null, future::complete); FetchLogResultForBucket result = future.get(10, TimeUnit.SECONDS).get(TABLE_BUCKET); - assertThat(result.failed()).isFalse(); + assertThat(result.failed()).as("Historical lookup failed: %s", result.getError()).isFalse(); return result.records(); } @@ -1341,7 +1411,7 @@ private static void assertHistoricalLookup( originalPartition), (lookupTimeNanos, lookupFileDownloaded) -> {}) .get(10, TimeUnit.SECONDS); - assertThat(result.failed()).isFalse(); + assertThat(result.failed()).as("Historical lookup failed: %s", result.getError()).isFalse(); assertThat(result.lookupValues()).hasSize(1); if (expectedRow == null) { assertThat(result.lookupValues().get(0)).isNull(); diff --git a/website/docs/maintenance/observability/monitor-metrics.md b/website/docs/maintenance/observability/monitor-metrics.md index fb88ab0a0a7..8aa9c640733 100644 --- a/website/docs/maintenance/observability/monitor-metrics.md +++ b/website/docs/maintenance/observability/monitor-metrics.md @@ -620,6 +620,22 @@ Some metrics might not be exposed when using other JVM implementations (e.g. IBM The cumulative number of lookup files evicted to enforce the shared TabletServer disk budget. Expiration, replacement, and explicit invalidation are excluded. Gauge + + kvLazyOpen + kvTabletOpenCount + The number of KvTablets currently in OPEN state (RocksDB loaded). + Gauge + + + kvTabletLazyCount + The number of KvTablets currently in LAZY state (RocksDB not loaded). + Gauge + + + kvTabletFailedCount + The number of KvTablets currently in FAILED state (open attempt failed, in backoff cooldown). + Gauge + logicalStorage logSize From ec1c488eee99b0210d6ba38947c902e7a1f50477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Thu, 8 Oct 2026 14:52:35 +0800 Subject: [PATCH 2/5] [server] Fix lazy KV idle tracking and close races Keep maintenance guards from refreshing idle access timestamps. Fence closing tablets and wait for open, release, callbacks and candidate cleanup before completing close or drop. Retain bucket ownership until cleanup completes and propagate shutdown failures before releasing shared native resources. Cover periodic snapshots, fenced opens, slow releases, shutdown and repeated drops with regression tests. Verified 202 related server tests and the Java 8 reactor build. --- .../org/apache/fluss/server/kv/KvManager.java | 41 +- .../org/apache/fluss/server/kv/KvTablet.java | 6 +- .../server/kv/KvTabletLazyLifecycle.java | 501 ++++++------------ .../apache/fluss/server/replica/Replica.java | 2 +- .../kv/KvIdleReleaseControllerTest.java | 1 + .../apache/fluss/server/kv/KvManagerTest.java | 2 +- .../server/kv/KvTabletLazyLifecycleTest.java | 193 ++++++- .../server/replica/ReplicaLazyOpenTest.java | 17 + 8 files changed, 412 insertions(+), 351 deletions(-) diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java index 80ff2c5e490..9ed7ed691f9 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java @@ -575,11 +575,8 @@ private void closeKvTablet(KvTablet kvTablet, KvCloseMode closeMode) { try { kvTablet.close(closeMode); } catch (Exception e) { - LOG.warn( - "Exception while closing kv tablet {} with mode {}.", - kvTablet.getTableBucket(), - closeMode, - e); + throw new KvStorageException( + "Failed to close KV tablet " + kvTablet.getTableBucket(), e); } } @@ -729,20 +726,28 @@ public Optional getKv(TableBucket tableBucket) { } public void dropKv(TableBucket tableBucket) { - inKvLock( - tableBucket, - () -> { - doDropKv(tableBucket); - return null; - }); - } - - private void doDropKv(TableBucket tableBucket) { - KvTablet dropKvTablet = currentKvs.remove(tableBucket); + KvTablet lazyTablet = + inKvLock( + tableBucket, + () -> { + KvTablet current = currentKvs.get(tableBucket); + if (current != null && !current.isLazyMode()) { + doDropKv(tableBucket, current); + return null; + } + return current; + }); + // Lazy open callbacks also need the bucket lock. Retain registry ownership while + // waiting, but do not hold that lock across lazy lifecycle cleanup. + doDropKv(tableBucket, lazyTablet); + } + + private void doDropKv(TableBucket tableBucket, @Nullable KvTablet dropKvTablet) { if (dropKvTablet != null) { TablePath tablePath = dropKvTablet.getTablePath(); try { dropKvTablet.drop(); + inKvLock(tableBucket, () -> currentKvs.remove(tableBucket, dropKvTablet)); if (dropKvTablet.getPartitionName() == null) { LOG.info( "Deleted kv bucket {} for table {} in file path {}.", @@ -898,9 +903,9 @@ public void registerKv(TableBucket tableBucket, KvTablet kvTablet) { }); } - /** Remove a KvTablet from the {@code currentKvs} registry (e.g. on drop). */ - public void unregisterKv(TableBucket tableBucket) { - inKvLock(tableBucket, () -> currentKvs.remove(tableBucket)); + /** Remove the expected tablet without unregistering a replacement for the same bucket. */ + public void unregisterKv(TableBucket tableBucket, KvTablet expectedTablet) { + inKvLock(tableBucket, () -> currentKvs.remove(tableBucket, expectedTablet)); } /** diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java index 9b9c852f95e..627abb9adae 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java @@ -388,7 +388,11 @@ boolean hasActiveResourceLeases() { void deleteLocalDirectory() { File dir = getKvTabletDir(); if (dir != null) { - FileUtils.deleteDirectoryQuietly(dir); + try { + FileUtils.deleteDirectory(dir); + } catch (IOException e) { + throw new KvStorageException("Failed to delete KV directory " + dir, e); + } } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTabletLazyLifecycle.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTabletLazyLifecycle.java index 07dc8f82e9d..c89ddd4d8ce 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTabletLazyLifecycle.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTabletLazyLifecycle.java @@ -69,6 +69,8 @@ enum LazyState { RELEASING, /** Lazy mode: last open attempt failed, in backoff cooldown. */ FAILED, + /** Closing: new requests are fenced while outstanding work drains. */ + CLOSING, /** * Terminal state: tablet is closed and will not be reopened. Local data may or may not have * been deleted depending on whether close or drop was called. @@ -174,8 +176,17 @@ public interface DropCallback { private @Nullable DropCallback dropCallback; private @Nullable ReleaseCallback releaseCallback; - /** Tracks post-open finalization that runs after OPEN becomes externally visible. */ - private boolean commitCallbackInFlight; + /** + * Includes open callbacks, candidate cleanup and idle release cleanup. Guarded by the state + * lock. + */ + private boolean operationInFlight; + + /** Retains ownership if cleaning up an uncommitted open fails. */ + private @Nullable KvTablet failedOpenTablet; + + /** Guarded by the termination monitor; prevents repeated drops from deleting a reused path. */ + private boolean localDirectoryDeleted; KvTabletLazyLifecycle(KvTablet tablet) { this.tablet = tablet; @@ -371,6 +382,7 @@ KvTablet.Guard acquireGuard() { // Fast path: try to pin without blocking KvTablet.Guard fast = tryAcquirePinAsGuard(); if (fast != null) { + touchAccessTimestamp(); return fast; } @@ -386,7 +398,7 @@ KvTablet.Guard acquireGuard() { touchAccessTimestamp(); return new KvTablet.Guard(tablet); } - if (lazyState == LazyState.CLOSED) { + if (lazyState == LazyState.CLOSED || lazyState == LazyState.CLOSING) { throw new KvStorageException( "KvTablet is closed for " + tablet.getTableBucket()); } @@ -427,7 +439,6 @@ private KvTablet.Guard tryAcquirePinAsGuard() { if (lazyState == LazyState.OPEN && !rejectNewPins) { activePins.incrementAndGet(); if (lazyState == LazyState.OPEN && !rejectNewPins) { - touchAccessTimestamp(); return new KvTablet.Guard(tablet); } decrementAndMaybeSignal(); @@ -543,8 +554,8 @@ private void ensureOpen() { // Fall through to LAZY — cooldown expired, retry open. case LAZY: - updateStateGauges(lazyState, LazyState.OPENING); - lazyState = LazyState.OPENING; + transitionTo(LazyState.OPENING); + operationInFlight = true; openGeneration++; long myGeneration = openGeneration; int myLeaderEpoch = @@ -556,8 +567,7 @@ private void ensureOpen() { try { doSlowOpen(myGeneration, myLeaderEpoch, myBucketEpoch, localDataExists); } catch (Throwable t) { - // doSlowOpen already called commitOpenResult() which transitions - // to FAILED, so no need to handle state transition here. + // doSlowOpen completes candidate cleanup before publishing failure. if (t instanceof RuntimeException) { throw (RuntimeException) t; } @@ -566,6 +576,7 @@ private void ensureOpen() { } return; + case CLOSING: case CLOSED: throw new KvStorageException( "KvTablet is closed for " + tablet.getTableBucket()); @@ -588,10 +599,11 @@ private void ensureOpen() { private void doSlowOpen( long myGeneration, int myLeaderEpoch, int myBucketEpoch, boolean localDataExists) throws Exception { - KvTablet newTablet = null; + KvTablet candidate = null; boolean committed = false; boolean semaphoreAcquired = false; - + Throwable failure = null; + boolean cleanupFailed = false; try { if (openSemaphore != null) { semaphoreAcquired = openSemaphore.tryAcquire(openTimeoutMs, TimeUnit.MILLISECONDS); @@ -600,114 +612,86 @@ private void doSlowOpen( "KvTablet open semaphore timeout for " + tablet.getTableBucket()); } } - - newTablet = openCallback.doOpen(localDataExists); - committed = - commitOpenResult(myGeneration, myLeaderEpoch, myBucketEpoch, newTablet, null); - if (!committed) { - throw new CancellationException( - "open result fenced by concurrent epoch/generation change"); - } - } catch (Exception e) { - commitOpenResult(myGeneration, myLeaderEpoch, myBucketEpoch, null, e); - throw e; - } finally { - if (!committed && newTablet != null) { - try { - newTablet.close(); - } catch (Exception ignored) { - } - // Clean up the directory created by the failed open attempt BEFORE - // releasing the semaphore, so that a subsequent open cannot start - // writing to the same canonical path while deletion is in progress. - File failedDir = newTablet.getKvTabletDir(); - if (failedDir != null) { - FileUtils.deleteDirectoryQuietly(failedDir); + candidate = openCallback.doOpen(localDataExists); + lazyStateLock.lock(); + try { + int leaderEpoch = leaderEpochSupplier != null ? leaderEpochSupplier.getAsInt() : -1; + int bucketEpoch = bucketEpochSupplier != null ? bucketEpochSupplier.getAsInt() : -1; + if (lazyState != LazyState.OPENING + || openGeneration != myGeneration + || leaderEpoch != myLeaderEpoch + || bucketEpoch != myBucketEpoch) { + throw new CancellationException( + "open result fenced by concurrent epoch/generation change"); } + tablet.installRocksDB(candidate); + committed = true; + transitionTo(LazyState.OPEN); + failureCount = 0; + lastAccessTimestamp = clock.milliseconds(); + rejectNewPins = false; + lazyStateChanged.signalAll(); + } finally { + lazyStateLock.unlock(); } - if (semaphoreAcquired) { - openSemaphore.release(); - } - } - } - - private boolean commitOpenResult( - long myGeneration, - int myLeaderEpoch, - int myBucketEpoch, - @Nullable KvTablet newTablet, - @Nullable Exception error) { - lazyStateLock.lock(); - try { - if (openGeneration != myGeneration) { - if (lazyState == LazyState.OPENING) { - transitionToFailed( - new CancellationException( - "open superseded by generation " + openGeneration), - true); + try { + if (commitCallback != null) { + commitCallback.onOpenCommitted(tablet); } - return false; - } - - if (error != null) { - transitionToFailed(error, false); - return false; - } - - // Epoch fencing - int currentLeaderEpoch = - leaderEpochSupplier != null ? leaderEpochSupplier.getAsInt() : -1; - int currentBucketEpoch = - bucketEpochSupplier != null ? bucketEpochSupplier.getAsInt() : -1; - if (currentLeaderEpoch != myLeaderEpoch || currentBucketEpoch != myBucketEpoch) { - transitionToFailed(new CancellationException("epoch changed during open"), true); - return false; - } - - // Success — absorb RocksDB state from the newly opened tablet. - // Note: we set OPEN and signal waiters before running commitCallback (which starts - // the snapshot manager). Writes during this brief window are safe — data is in - // RocksDB and the WAL. The commitCallbackInFlight flag prevents release until the - // callback completes. - tablet.installRocksDB(newTablet); - lazyState = LazyState.OPEN; - failureCount = 0; - lastAccessTimestamp = clock.milliseconds(); - rejectNewPins = false; - commitCallbackInFlight = true; - lazyStateChanged.signalAll(); - } finally { - lazyStateLock.unlock(); - } - - try { - updateStateGauges(LazyState.OPENING, LazyState.OPEN); - - // Invoke commit callback outside the lock to avoid blocking acquireGuard() callers. - if (commitCallback != null) { - commitCallback.onOpenCommitted(tablet); + } catch (Exception e) { + LOG.warn("Post-open commit callback failed for {}", tablet.getTableBucket(), e); } - } catch (Exception e) { - LOG.warn( - "Post-open commit callback failed for {}, keeping tablet OPEN", - tablet.getTableBucket(), - e); + } catch (Exception | Error e) { + failure = e; + throw e; } finally { - lazyStateLock.lock(); try { - commitCallbackInFlight = false; - lazyStateChanged.signalAll(); + if (!committed && candidate != null) { + candidate.close(); + File dir = candidate.getKvTabletDir(); + if (dir != null) { + FileUtils.deleteDirectory(dir); + } + } + } catch (Exception | Error e) { + cleanupFailed = true; + if (failure != null) { + failure.addSuppressed(e); + } else { + failure = e; + throw e; + } } finally { - lazyStateLock.unlock(); + if (semaphoreAcquired) { + openSemaphore.release(); + } + lazyStateLock.lock(); + try { + if (cleanupFailed) { + failedOpenTablet = candidate; + rejectNewPins = true; + transitionTo(LazyState.CLOSING); + } else if (!committed && lazyState != LazyState.CLOSING) { + transitionToFailed(failure, failure instanceof CancellationException); + } + operationInFlight = false; + lazyStateChanged.signalAll(); + } finally { + lazyStateLock.unlock(); + } } } - return true; + } + + /** Must be called while holding {@code lazyStateLock}. */ + private void transitionTo(LazyState state) { + updateStateGauges(lazyState, state); + lazyState = state; } /** Must be called while holding {@code lazyStateLock}. */ private void transitionToFailed(Throwable cause, boolean resetCount) { - updateStateGauges(LazyState.OPENING, LazyState.FAILED); - lazyState = LazyState.FAILED; + transitionTo(LazyState.FAILED); failedTimestamp = clock.milliseconds(); if (resetCount) { failureCount = 0; @@ -738,76 +722,58 @@ boolean canRelease(long closeIdleIntervalMs, long nowMs) { * @return true if release succeeded, false if aborted */ boolean releaseKv() { - // Pre-check state outside lock as a fast-path rejection - if (lazyState != LazyState.OPEN) { - return false; - } - lazyStateLock.lock(); try { - if (lazyState != LazyState.OPEN) { + if (lazyState != LazyState.OPEN || operationInFlight) { return false; } - - // Transition to RELEASING — write rejectNewPins before lazyState so that - // tryAcquirePinAsGuard() sees the reject flag if it reads OPEN. rejectNewPins = true; - lazyState = LazyState.RELEASING; - - if (!drainPins()) { - LOG.warn("{} release drain timeout, aborting release", tablet.getTableBucket()); - transitionBackToOpen(); - return false; - } - - // Re-check flush progress only after blocking new pins and draining - // all in-flight operations. This closes the race where a writer acquired - // a guard before RELEASING but appended after an earlier LEO check. - long logEndOffset = tablet.logTablet.localLogEndOffset(); - if (tablet.getFlushedLogOffset() < logEndOffset || tablet.hasActiveResourceLeases()) { - transitionBackToOpen(); - return false; - } - - if (!awaitCommitCallbackCompletion()) { - LOG.warn( - "{} release waited too long for open commit finalization, aborting release", - tablet.getTableBucket()); - transitionBackToOpen(); + transitionTo(LazyState.RELEASING); + operationInFlight = true; + if (!drainPins() + || lazyState == LazyState.CLOSING + || tablet.getFlushedLogOffset() < tablet.logTablet.localLogEndOffset() + || tablet.hasActiveResourceLeases()) { + if (lazyState == LazyState.RELEASING) { + transitionBackToOpen(); + } + operationInFlight = false; + lazyStateChanged.signalAll(); return false; } - - // Cache metadata AFTER drain — all pinned operations have completed, - // so rowCount reflects the latest flushed state. cachedRowCount = tablet.currentRowCount(); cachedFlushedLogOffset = tablet.getFlushedLogOffset(); } finally { lazyStateLock.unlock(); } - // All pins drained, safe to close boolean releaseSucceeded = false; + boolean nativeCloseStarted = false; try { if (releaseCallback != null) { releaseCallback.doRelease(tablet); } + nativeCloseStarted = true; tablet.detachRocksDB(); releaseSucceeded = true; } catch (Exception e) { - LOG.warn( - "{} release callback failed, rolling back to OPEN", tablet.getTableBucket(), e); + LOG.warn("Failed to release KV tablet {}", tablet.getTableBucket(), e); } finally { lazyStateLock.lock(); try { - if (releaseSucceeded) { - lazyState = LazyState.LAZY; - hasLocalData = true; - updateStateGauges(LazyState.OPEN, LazyState.LAZY); - rejectNewPins = false; - lazyStateChanged.signalAll(); - } else { - transitionBackToOpen(); + if (lazyState == LazyState.RELEASING) { + if (releaseSucceeded) { + transitionTo(LazyState.LAZY); + hasLocalData = true; + rejectNewPins = false; + } else if (!nativeCloseStarted) { + transitionBackToOpen(); + } else { + transitionTo(LazyState.CLOSING); + } } + operationInFlight = false; + lazyStateChanged.signalAll(); } finally { lazyStateLock.unlock(); } @@ -815,73 +781,70 @@ boolean releaseKv() { return releaseSucceeded; } - // ---- Drop logic ---- - - /** - * Full-state cleanup for lazy mode: handles all states, waits for in-progress operations, then - * transitions to CLOSED. Called by Replica.dropKv(). - */ + /** Close resources and remove local data only after outstanding operations finish. */ void dropKvLazy() { - lazyStateLock.lock(); - try { - switch (lazyState) { - case CLOSED: - break; - - case LAZY: - case FAILED: - tablet.deleteLocalDirectory(); - break; - - case OPENING: - // Fence the in-progress open so commitOpenResult() will reject it. - // If awaitStateExit times out (open thread stuck on semaphore or slow I/O), - // the open thread will eventually call commitOpenResult(), find the generation - // mismatch, and transition to FAILED. The final CLOSED transition at the - // bottom of this method will override FAILED. The open thread's finally block - // in doSlowOpen will close the newTablet, preventing resource leaks. - openGeneration++; - awaitStateExit(LazyState.OPENING, openTimeoutMs * 2); - if (lazyState == LazyState.OPEN) { - rejectNewPins = true; - awaitCommitCallbackCompletion(); - drainPinsForDrop(); - dropOpenTablet(); - } else if (lazyState != LazyState.OPENING) { - tablet.deleteLocalDirectory(); - } - break; - - case OPEN: - rejectNewPins = true; - awaitCommitCallbackCompletion(); - drainPinsForDrop(); - dropOpenTablet(); - break; + terminate(KvCloseMode.DISCARD_UNPERSISTED_STATE, true); + } - case RELEASING: - awaitStateExit(LazyState.RELEASING, releaseDrainTimeoutMs * 2); - if (lazyState == LazyState.OPEN) { - rejectNewPins = true; - awaitCommitCallbackCompletion(); - drainPinsForDrop(); - dropOpenTablet(); - } else if (lazyState != LazyState.RELEASING) { - tablet.deleteLocalDirectory(); - } - break; + void closeKvLazy(KvCloseMode closeMode) { + terminate(closeMode, false); + } - default: - throw new IllegalStateException("Unexpected state in dropKvLazy: " + lazyState); + /** Serializes close callers without holding the state lock during callbacks or native I/O. */ + private synchronized void terminate(KvCloseMode closeMode, boolean deleteDirectory) { + if (lazyState == LazyState.CLOSED) { + if (deleteDirectory && !localDirectoryDeleted) { + if (dropCallback != null) { + dropCallback.doDrop(tablet); + } + tablet.deleteLocalDirectory(); + localDirectoryDeleted = true; + } + return; + } + lazyStateLock.lock(); + try { + rejectNewPins = true; + openGeneration++; + transitionTo(LazyState.CLOSING); + lazyStateChanged.signalAll(); + // A request timeout cannot transfer ownership of native resources or the local path. + while (operationInFlight || activePins.get() > 0) { + lazyStateChanged.awaitUninterruptibly(); } + } finally { + lazyStateLock.unlock(); + } - LazyState stateBeforeClose = lazyState; - if (stateBeforeClose != LazyState.CLOSED) { - updateStateGauges(stateBeforeClose, LazyState.CLOSED); + if (deleteDirectory) { + if (dropCallback != null) { + dropCallback.doDrop(tablet); + } + } else if (releaseCallback != null) { + releaseCallback.doRelease(tablet); + } + if (failedOpenTablet != null) { + try { + failedOpenTablet.close(); + File dir = failedOpenTablet.getKvTabletDir(); + if (dir != null) { + FileUtils.deleteDirectory(dir); + } + failedOpenTablet = null; + } catch (Exception e) { + throw new KvStorageException( + "Failed to clean up fenced open for " + tablet.getTableBucket(), e); } - lazyState = LazyState.CLOSED; + } + tablet.detachRocksDB(closeMode); + if (deleteDirectory) { + tablet.deleteLocalDirectory(); + localDirectoryDeleted = true; + } + lazyStateLock.lock(); + try { + transitionTo(LazyState.CLOSED); hasLocalData = false; - rejectNewPins = false; failureCount = 0; failedTimestamp = 0; lastFailureCause = null; @@ -891,15 +854,6 @@ void dropKvLazy() { } } - private void drainPinsForDrop() { - if (!drainPins()) { - LOG.warn( - "{} drop drain timeout with {} active pins, proceeding with close", - tablet.getTableBucket(), - activePins.get()); - } - } - private boolean drainPins() { long deadline = clock.milliseconds() + releaseDrainTimeoutMs; while (activePins.get() > 0) { @@ -917,116 +871,9 @@ private boolean drainPins() { return true; } - /** Waits under {@code lazyStateLock} for post-open finalization to complete. */ - private boolean awaitCommitCallbackCompletion() { - long deadline = clock.milliseconds() + releaseDrainTimeoutMs; - while (commitCallbackInFlight) { - long remaining = deadline - clock.milliseconds(); - if (remaining <= 0) { - return false; - } - try { - if (!lazyStateChanged.await(remaining, TimeUnit.MILLISECONDS)) { - return false; - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return false; - } - } - return true; - } - - /** - * Wait (under lazyStateLock) until {@code lazyState} is no longer {@code state}, or timeout. - */ - private void awaitStateExit(LazyState state, long timeoutMs) { - long deadlineNanos = clock.nanoseconds() + TimeUnit.MILLISECONDS.toNanos(timeoutMs); - while (lazyState == state) { - try { - long remainNanos = deadlineNanos - clock.nanoseconds(); - if (remainNanos <= 0 - || !lazyStateChanged.await(remainNanos, TimeUnit.NANOSECONDS)) { - break; - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; - } - } - } - - private void doDropViaCallback() { - if (dropCallback != null) { - dropCallback.doDrop(tablet); - } - } - - void closeKvLazy(KvCloseMode closeMode) { - lazyStateLock.lock(); - try { - switch (lazyState) { - case CLOSED: - return; - case LAZY: - case FAILED: - break; - case OPENING: - openGeneration++; - awaitStateExit(LazyState.OPENING, openTimeoutMs * 2); - if (lazyState == LazyState.OPEN) { - rejectNewPins = true; - awaitCommitCallbackCompletion(); - drainPins(); - tablet.detachRocksDB(closeMode); - } - break; - case OPEN: - rejectNewPins = true; - awaitCommitCallbackCompletion(); - drainPins(); - tablet.detachRocksDB(closeMode); - break; - case RELEASING: - awaitStateExit(LazyState.RELEASING, releaseDrainTimeoutMs * 2); - if (lazyState == LazyState.OPEN) { - rejectNewPins = true; - awaitCommitCallbackCompletion(); - drainPins(); - tablet.detachRocksDB(closeMode); - } - break; - default: - throw new IllegalStateException( - "Unexpected state in closeKvLazy: " + lazyState); - } - - LazyState stateBeforeClose = lazyState; - if (stateBeforeClose != LazyState.CLOSED) { - updateStateGauges(stateBeforeClose, LazyState.CLOSED); - } - lazyState = LazyState.CLOSED; - hasLocalData = false; - rejectNewPins = false; - failureCount = 0; - failedTimestamp = 0; - lastFailureCause = null; - lazyStateChanged.signalAll(); - } finally { - lazyStateLock.unlock(); - } - } - - /** Must be called while holding {@code lazyStateLock} and after pins have drained. */ - private void dropOpenTablet() { - doDropViaCallback(); - tablet.detachRocksDB(KvCloseMode.DISCARD_UNPERSISTED_STATE); - tablet.deleteLocalDirectory(); - } - /** Must be called while holding {@code lazyStateLock}. */ private void transitionBackToOpen() { - lazyState = LazyState.OPEN; + transitionTo(LazyState.OPEN); rejectNewPins = false; lazyStateChanged.signalAll(); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index 4ff9f24e2c3..5b8e1edb8c5 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -844,7 +844,7 @@ private void dropKv() { kv.dropKvLazy(); // Remove sentinel from KvManager registry checkNotNull(kvManager); - kvManager.unregisterKv(tableBucket); + kvManager.unregisterKv(tableBucket, kv); } else { bucketMetricGroup.unregisterRocksDBStatistics(); checkNotNull(kvManager); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvIdleReleaseControllerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvIdleReleaseControllerTest.java index 8909d059359..0bfda40c03a 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvIdleReleaseControllerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvIdleReleaseControllerTest.java @@ -327,5 +327,6 @@ void testContinuesAfterReleaseFailure() throws Exception { assertThat(tablet1.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); // tablet2 should still be released despite tablet1's failure assertThat(tablet2.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); + tablet1.getLifecycle().setReleaseCallback(kv -> {}); } } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvManagerTest.java index 730cdc111ef..1df2d336a42 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvManagerTest.java @@ -833,7 +833,7 @@ void testUnregisterKv(String partitionName) throws Exception { KvTablet tablet = getOrCreateKv(tablePath1, partitionName, tableBucket1); assertThat(kvManager.getKv(tableBucket1)).isPresent(); - kvManager.unregisterKv(tableBucket1); + kvManager.unregisterKv(tableBucket1, tablet); assertThat(kvManager.getKv(tableBucket1)).isNotPresent(); tablet.close(); } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletLazyLifecycleTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletLazyLifecycleTest.java index b46e5add3aa..fc834605f27 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletLazyLifecycleTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletLazyLifecycleTest.java @@ -38,6 +38,7 @@ import org.apache.fluss.server.metrics.group.TabletServerMetricGroup; import org.apache.fluss.server.metrics.group.TestingMetricGroups; import org.apache.fluss.server.storage.LocalDiskManager; +import org.apache.fluss.server.utils.ResourceGuard; import org.apache.fluss.server.zk.NOPErrorHandler; import org.apache.fluss.server.zk.ZooKeeperClient; import org.apache.fluss.server.zk.ZooKeeperExtension; @@ -45,6 +46,7 @@ import org.apache.fluss.testutils.common.CommonTestUtils; import org.apache.fluss.types.RowType; import org.apache.fluss.utils.ByteArraySlice; +import org.apache.fluss.utils.clock.Clock; import org.apache.fluss.utils.clock.ManualClock; import org.apache.fluss.utils.clock.SystemClock; import org.apache.fluss.utils.concurrent.FlussScheduler; @@ -55,6 +57,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.io.File; import java.time.Duration; @@ -64,6 +68,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -169,7 +174,12 @@ private KvTablet createSentinel() { return tablet; } - private KvTabletLazyLifecycle configureLifecycle(KvTablet sentinel, ManualClock clock) { + private KvTabletLazyLifecycle configureLifecycle(KvTablet sentinel, Clock clock) { + return configureLifecycle(sentinel, clock, KvManager.RELEASE_DRAIN_TIMEOUT_MS); + } + + private KvTabletLazyLifecycle configureLifecycle( + KvTablet sentinel, Clock clock, long drainTimeout) { KvTabletLazyLifecycle lifecycle = sentinel.getLifecycle(); lifecycle.configureLazyOpen( clock, @@ -177,7 +187,7 @@ private KvTabletLazyLifecycle configureLifecycle(KvTablet sentinel, ManualClock kvManager.getOpenTimeoutMs(), kvManager.getFailedBackoffBaseMs(), kvManager.getFailedBackoffMaxMs(), - KvManager.RELEASE_DRAIN_TIMEOUT_MS); + drainTimeout); lifecycle.setLeaderEpochSupplier(() -> 1); lifecycle.setBucketEpochSupplier(() -> 1); lifecycle.setTabletDirSupplier( @@ -210,6 +220,183 @@ private void flushAndAwait(KvTablet tablet) throws Exception { "KV operation did not complete"); } + @Test + void testMaintenanceDoesNotRefreshIdleTime() throws Exception { + ManualClock clock = new ManualClock(System.currentTimeMillis()); + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); + try (KvTablet.Guard ignored = sentinel.acquireGuard()) { + // Open through a business request. + } + long lastAccess = sentinel.getLastAccessTimestamp(); + clock.advanceTime(10, TimeUnit.MINUTES); + sentinel.getGuardedExecutor().execute(() -> {}); + assertThat(sentinel.getLastAccessTimestamp()).isEqualTo(lastAccess); + assertThat(lifecycle.canRelease(TimeUnit.MINUTES.toMillis(10), clock.milliseconds())) + .isTrue(); + try (KvTablet.Guard ignored = sentinel.acquireGuard()) { + assertThat(sentinel.getLastAccessTimestamp()).isEqualTo(clock.milliseconds()); + } + assertThat(lifecycle.canRelease(TimeUnit.MINUTES.toMillis(10), clock.milliseconds())) + .isFalse(); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testCloseWaitsForSlowRelease(boolean delete) throws Exception { + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = + configureLifecycle(sentinel, SystemClock.getInstance(), 20); + try (KvTablet.Guard ignored = sentinel.acquireGuard()) { + // Open an empty tablet eligible for idle release. + } + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch finish = new CountDownLatch(1); + lifecycle.setReleaseCallback( + kv -> { + entered.countDown(); + try { + finish.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + }); + FutureTask release = new FutureTask<>(sentinel::releaseKv); + FutureTask drop = + new FutureTask<>( + () -> { + if (delete) { + sentinel.dropKvLazy(); + } else { + sentinel.close(); + } + return null; + }); + new Thread(release, "slow-idle-release").start(); + try { + assertThat(entered.await(5, TimeUnit.SECONDS)).isTrue(); + new Thread(drop, "drop-slow-release").start(); + assertThatThrownBy(() -> drop.get(200, TimeUnit.MILLISECONDS)) + .isInstanceOf(TimeoutException.class); + } finally { + finish.countDown(); + release.get(5, TimeUnit.SECONDS); + drop.get(5, TimeUnit.SECONDS); + } + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); + assertThat(sentinel.getRocksDBKv()).isNull(); + assertThat(sentinel.getKvTabletDir().exists()).isEqualTo(!delete); + assertThatThrownBy(sentinel::acquireGuard).isInstanceOf(KvStorageException.class); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testCloseWaitsForFencedOpenCleanup(boolean shutdown) throws Exception { + KvTablet sentinel = createSentinel(); + KvTabletLazyLifecycle lifecycle = + configureLifecycle(sentinel, SystemClock.getInstance(), 20); + CountDownLatch prepared = new CountDownLatch(1); + CountDownLatch returnCandidate = new CountDownLatch(1); + AtomicReference candidate = new AtomicReference<>(); + AtomicReference lease = new AtomicReference<>(); + lifecycle.setOpenCallback( + hasLocal -> { + KvTablet opened = + kvManager.createKvTabletUnregistered( + physicalPath, + tableBucket, + logTablet, + KvFormat.COMPACTED, + new TestingSchemaGetter( + new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), + new TableConfig(new Configuration()), + DEFAULT_COMPRESSION, + null); + candidate.set(opened); + lease.set(opened.getRocksDBKv().getResourceGuard().acquireResource()); + prepared.countDown(); + returnCandidate.await(); + return opened; + }); + FutureTask open = + new FutureTask<>( + () -> { + assertThatThrownBy(sentinel::acquireGuard) + .isInstanceOf(RuntimeException.class); + return null; + }); + FutureTask close = + new FutureTask<>( + () -> { + if (shutdown) { + kvManager.shutdown(); + } else { + kvManager.dropKv(tableBucket); + } + return null; + }); + new Thread(open, "fenced-open").start(); + try { + assertThat(prepared.await(5, TimeUnit.SECONDS)).isTrue(); + new Thread(close, "close-fenced-open").start(); + CommonTestUtils.waitUntil( + () -> lifecycle.getLazyState() == KvTabletLazyLifecycle.LazyState.CLOSING, + Duration.ofSeconds(5), + "Close did not fence opening tablet"); + returnCandidate.countDown(); + CommonTestUtils.waitUntil( + () -> candidate.get().getRocksDBKv().getResourceGuard().isClosed(), + Duration.ofSeconds(5), + "Candidate cleanup did not start"); + assertThatThrownBy(() -> close.get(200, TimeUnit.MILLISECONDS)) + .isInstanceOf(TimeoutException.class); + assertThat(kvManager.getKv(tableBucket)).contains(sentinel); + assertThatThrownBy(() -> kvManager.registerKv(tableBucket, sentinel)) + .isInstanceOf(IllegalStateException.class); + } finally { + returnCandidate.countDown(); + if (lease.get() != null) { + lease.get().close(); + } + open.get(5, TimeUnit.SECONDS); + close.get(5, TimeUnit.SECONDS); + } + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); + assertThat(sentinel.getRocksDBKv()).isNull(); + assertThat(sentinel.getKvTabletDir()).doesNotExist(); + if (shutdown) { + kvManager = null; + } else { + assertThat(kvManager.getKv(tableBucket)).isEmpty(); + KvTablet replacement = createSentinel(); + configureLifecycle(replacement, SystemClock.getInstance()); + try (KvTablet.Guard ignored = replacement.acquireGuard()) { + assertThat(replacement.getKvTabletDir()).isDirectory(); + } + } + } + + @Test + void testDropAfterCloseDeletesLocalDataOnlyOnce() throws Exception { + KvTablet sentinel = createSentinel(); + configureLifecycle(sentinel, SystemClock.getInstance()); + try (KvTablet.Guard ignored = sentinel.acquireGuard()) { + assertThat(sentinel.getKvTabletDir()).isDirectory(); + } + sentinel.close(); + assertThat(sentinel.getKvTabletDir()).isDirectory(); + sentinel.dropKvLazy(); + assertThat(sentinel.getKvTabletDir()).doesNotExist(); + kvManager.unregisterKv(tableBucket, sentinel); + KvTablet replacement = createSentinel(); + configureLifecycle(replacement, SystemClock.getInstance()); + try (KvTablet.Guard ignored = replacement.acquireGuard()) { + sentinel.dropKvLazy(); + assertThat(replacement.getKvTabletDir()).isDirectory(); + } + } + // ---- Tests ---- @Test @@ -618,7 +805,7 @@ void testDropWaitsForCommitCallbackCompletion() throws Exception { Duration.ofSeconds(5), "Timed out waiting for drop thread to block"); assertThat(dropTask.isDone()).isFalse(); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSING); allowCommitCallbackFinish.countDown(); openThread.join(TimeUnit.SECONDS.toMillis(5)); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaLazyOpenTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaLazyOpenTest.java index 79e991c622f..daac035e4d7 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaLazyOpenTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaLazyOpenTest.java @@ -76,6 +76,23 @@ protected Configuration getServerConf() { return conf; } + @Test + void testPeriodicSnapshotDoesNotKeepIdleTabletOpen() throws Exception { + Replica replica = makeLazyLeaderReplica(); + replica.lookups(Collections.singletonList(key(1))); + KvTablet tablet = checkNotNull(replica.getKvTablet()); + long lastAccess = tablet.getLastAccessTimestamp(); + manualClock.advanceTime(Duration.ofMinutes(10)); + checkNotNull(replica.getKvSnapshotManager()).triggerSnapshot(); + assertThat(tablet.getLastAccessTimestamp()).isEqualTo(lastAccess); + assertThat(tablet.canRelease(Duration.ofMinutes(10).toMillis(), manualClock.milliseconds())) + .isTrue(); + replica.lookups(Collections.singletonList(key(1))); + assertThat(tablet.getLastAccessTimestamp()).isEqualTo(manualClock.milliseconds()); + assertThat(tablet.canRelease(Duration.ofMinutes(10).toMillis(), manualClock.milliseconds())) + .isFalse(); + } + @Test void testFirstLookupFlushesRecoveredRecordsBeyondHighWatermark() throws Exception { Replica replica = From 559e3d0b612fdd6b710b39ba4877142430d94d31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Thu, 8 Oct 2026 14:52:41 +0800 Subject: [PATCH 3/5] [build] Remove duplicate POM declarations for Java 8 Merge OSS jar and test-jar goals into one plugin declaration while preserving the multi-release manifest. Remove the duplicate ordinary Flink connector-base dependency and retain its provided and test-jar dependencies. Validated clean install with JDK 8 and the java8 profile across all 48 reactor modules. --- fluss-filesystems/fluss-fs-oss/pom.xml | 13 +------------ fluss-flink/fluss-flink-common/pom.xml | 7 ------- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/fluss-filesystems/fluss-fs-oss/pom.xml b/fluss-filesystems/fluss-fs-oss/pom.xml index 7ebd7c4dd14..4cd62233b54 100644 --- a/fluss-filesystems/fluss-fs-oss/pom.xml +++ b/fluss-filesystems/fluss-fs-oss/pom.xml @@ -124,6 +124,7 @@ jar + test-jar @@ -237,18 +238,6 @@ - - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - diff --git a/fluss-flink/fluss-flink-common/pom.xml b/fluss-flink/fluss-flink-common/pom.xml index dcb81c741b5..7c0918dce61 100644 --- a/fluss-flink/fluss-flink-common/pom.xml +++ b/fluss-flink/fluss-flink-common/pom.xml @@ -117,13 +117,6 @@ test - - org.apache.flink - flink-connector-base - ${flink.minor.version} - test - - org.apache.flink flink-connector-base From 82d97a183e4d5286653b49520ad3c5a90831c2bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Thu, 8 Oct 2026 16:02:48 +0800 Subject: [PATCH 4/5] [test] Wait for bucket readiness before bounded split discovery Wait for each replica and its metadata after creating the primary-key table, before the bounded enumerator requests stopping offsets. Table creation can finish before tablet servers are ready to serve ListOffsets. Validated FlinkSourceEnumeratorTest with Maven verify: all 30 cases passed in both test executions. --- .../flink/source/enumerator/FlinkSourceEnumeratorTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java index 43764059341..7df75a24276 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java @@ -214,7 +214,10 @@ void testBoundedPkTableEmitsKvBatchSplits() throws Throwable { @Test void testBoundedPkTableEmitsSnapshotSplitsByDefault() throws Throwable { - createTable(DEFAULT_TABLE_PATH, DEFAULT_PK_TABLE_DESCRIPTOR); + long tableId = createTable(DEFAULT_TABLE_PATH, DEFAULT_PK_TABLE_DESCRIPTOR); + for (int bucketId = 0; bucketId < DEFAULT_BUCKET_NUM; bucketId++) { + FLUSS_CLUSTER_EXTENSION.waitUntilAllReplicaReady(new TableBucket(tableId, bucketId)); + } int numSubtasks = DEFAULT_BUCKET_NUM; try (MockSplitEnumeratorContext context = new MockSplitEnumeratorContext<>(numSubtasks)) { From e71b5561f385bd6ad07e4e4c3b75ef16cfeddd1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Thu, 8 Oct 2026 18:11:40 +0800 Subject: [PATCH 5/5] [kv] Simplify lazy tablet ownership and recovery --- .../server/kv/KvIdleReleaseController.java | 152 -- .../org/apache/fluss/server/kv/KvManager.java | 219 ++- .../org/apache/fluss/server/kv/KvTablet.java | 208 +-- .../server/kv/KvTabletLazyLifecycle.java | 554 ++---- .../apache/fluss/server/replica/Replica.java | 309 +--- .../fluss/server/replica/ReplicaManager.java | 44 +- .../kv/KvIdleReleaseControllerTest.java | 332 ---- .../apache/fluss/server/kv/KvManagerTest.java | 680 ++++++-- .../server/kv/KvTabletLazyLifecycleTest.java | 1482 ----------------- .../server/replica/ReplicaLazyOpenTest.java | 244 +-- .../fluss/server/replica/ReplicaTest.java | 28 +- .../HistoricalPartitionManagerTest.java | 5 +- 12 files changed, 934 insertions(+), 3323 deletions(-) delete mode 100644 fluss-server/src/main/java/org/apache/fluss/server/kv/KvIdleReleaseController.java delete mode 100644 fluss-server/src/test/java/org/apache/fluss/server/kv/KvIdleReleaseControllerTest.java delete mode 100644 fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletLazyLifecycleTest.java diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvIdleReleaseController.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvIdleReleaseController.java deleted file mode 100644 index e290696f8d8..00000000000 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvIdleReleaseController.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.server.kv; - -import org.apache.fluss.annotation.VisibleForTesting; -import org.apache.fluss.utils.clock.Clock; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.util.Collection; -import java.util.Comparator; -import java.util.List; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -/** - * Periodically checks OPEN KvTablets and releases idle ones back to LAZY state. Tablets are sorted - * by last access time (LRU). Operates directly on {@link KvTablet} — no dependency on Replica - * layer. - */ -public class KvIdleReleaseController implements Closeable { - - private static final Logger LOG = LoggerFactory.getLogger(KvIdleReleaseController.class); - - private final ScheduledExecutorService scheduler; - private final Supplier> tabletSupplier; - private final Clock clock; - - private final long checkIntervalMs; - private final long idleIntervalMs; - - private volatile ScheduledFuture scheduledTask; - - public KvIdleReleaseController( - ScheduledExecutorService scheduler, - Supplier> tabletSupplier, - Clock clock, - long checkIntervalMs, - long idleIntervalMs) { - this.scheduler = scheduler; - this.tabletSupplier = tabletSupplier; - this.clock = clock; - this.checkIntervalMs = checkIntervalMs; - this.idleIntervalMs = idleIntervalMs; - } - - public void start() { - scheduledTask = - scheduler.scheduleWithFixedDelay( - this::checkAndRelease, - checkIntervalMs, - checkIntervalMs, - TimeUnit.MILLISECONDS); - LOG.info( - "KvIdleReleaseController started: checkInterval={}ms, idleInterval={}ms", - checkIntervalMs, - idleIntervalMs); - } - - /** - * Scans all open tablets, selects those idle beyond the configured threshold, and releases them - * in LRU order (coldest first). - */ - @VisibleForTesting - void checkAndRelease() { - try { - Collection tablets = tabletSupplier.get(); - long now = clock.milliseconds(); - - // Snapshot timestamp once per tablet to avoid double-read of volatile field - List idleCandidates = - tablets.stream() - .map(t -> new ReleaseCandidate(t, t.getLastAccessTimestamp())) - .filter(c -> now - c.lastAccessTimestamp > idleIntervalMs) - .sorted(Comparator.comparingLong(c -> c.lastAccessTimestamp)) - .collect(Collectors.toList()); - - if (idleCandidates.isEmpty()) { - LOG.debug("Idle release round: no idle tablets found"); - return; - } - - LOG.info("Idle release round: open={}, idle={}", tablets.size(), idleCandidates.size()); - - int released = 0; - for (ReleaseCandidate candidate : idleCandidates) { - KvTablet tablet = candidate.tablet; - - if (tablet.canRelease(idleIntervalMs, now)) { - try { - boolean success = tablet.releaseKv(); - if (success) { - released++; - LOG.debug( - "Released KvTablet for {} (idle {}ms)", - tablet.getTableBucket(), - now - candidate.lastAccessTimestamp); - } - } catch (Exception e) { - LOG.warn("Failed to release KvTablet for {}", tablet.getTableBucket(), e); - } - } - } - - if (released > 0) { - LOG.info( - "Idle release round complete: released={}/{}", - released, - idleCandidates.size()); - } - } catch (Exception e) { - LOG.error("Error during idle release check", e); - } - } - - @Override - public void close() { - if (scheduledTask != null) { - scheduledTask.cancel(false); - } - } - - private static class ReleaseCandidate { - final KvTablet tablet; - final long lastAccessTimestamp; - - ReleaseCandidate(KvTablet tablet, long lastAccessTimestamp) { - this.tablet = tablet; - this.lastAccessTimestamp = lastAccessTimestamp; - } - } -} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java index 9ed7ed691f9..53b894a51d2 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java @@ -130,20 +130,9 @@ public static RateLimiter getDefaultRateLimiter() { return DEFAULT_RATE_LIMITER; } - // ---- KV lazy open internal constants ---- - // Not exposed as ConfigOptions because these are internal tuning parameters that should - // rarely need adjustment. If operational experience shows otherwise, promote to ConfigOptions - // with keys like "kv.lazy-open.max-concurrent-opens", "kv.lazy-open.open-timeout", etc. private static final int LAZY_OPEN_MAX_CONCURRENT_OPENS = 10; - private static final long LAZY_OPEN_TIMEOUT_MS = 300_000; - private static final long LAZY_OPEN_FAILED_BACKOFF_BASE_MS = 5_000; - private static final long LAZY_OPEN_FAILED_BACKOFF_MAX_MS = 300_000; - /** - * Timeout for draining pins during release in milliseconds. Public because {@link - * org.apache.fluss.server.replica.Replica} (different package) needs this value. - */ - public static final long RELEASE_DRAIN_TIMEOUT_MS = 5_000; + static final long RELEASE_DRAIN_TIMEOUT_MS = 5_000; private final LogManager logManager; private final LocalDiskManager localDiskManager; @@ -190,12 +179,8 @@ public static RateLimiter getDefaultRateLimiter() { private final KvFlushScheduler kvFlushScheduler; - // ---- KV lazy open configuration ---- private final boolean lazyOpenEnabled; private final @Nullable Semaphore openSemaphore; - private final long openTimeoutMs; - private final long failedBackoffBaseMs; - private final long failedBackoffMaxMs; private volatile boolean isShutdown = false; @@ -217,19 +202,8 @@ private KvManager( this.zkClient = zkClient; this.clock = clock; this.remoteKvDir = FlussPaths.remoteKvDir(conf); - // KV lazy open configuration this.lazyOpenEnabled = conf.get(ConfigOptions.KV_LAZY_OPEN_ENABLED); - if (lazyOpenEnabled) { - this.openSemaphore = new Semaphore(LAZY_OPEN_MAX_CONCURRENT_OPENS); - this.openTimeoutMs = LAZY_OPEN_TIMEOUT_MS; - this.failedBackoffBaseMs = LAZY_OPEN_FAILED_BACKOFF_BASE_MS; - this.failedBackoffMaxMs = LAZY_OPEN_FAILED_BACKOFF_MAX_MS; - } else { - this.openSemaphore = null; - this.openTimeoutMs = 0; - this.failedBackoffBaseMs = 0; - this.failedBackoffMaxMs = 0; - } + this.openSemaphore = lazyOpenEnabled ? new Semaphore(LAZY_OPEN_MAX_CONCURRENT_OPENS) : null; this.remoteFileSystem = remoteKvDir.getFileSystem(); this.serverMetricGroup = tabletServerMetricGroup; this.sharedRocksDBRateLimiter = createSharedRateLimiter(conf); @@ -388,24 +362,30 @@ public boolean isLazyOpenEnabled() { return lazyOpenEnabled; } - /** Returns the semaphore limiting concurrent lazy opens, or null when disabled. */ - public @Nullable Semaphore getOpenSemaphore() { - return openSemaphore; - } - - /** Returns the lazy-open wait timeout in milliseconds. */ - public long getOpenTimeoutMs() { - return openTimeoutMs; - } - - /** Returns the initial retry backoff after a failed lazy open. */ - public long getFailedBackoffBaseMs() { - return failedBackoffBaseMs; + /** Releases idle tablets without changing their registry ownership or deleting local SSTs. */ + public void releaseIdleTablets(long idleTimeoutMs, long nowMs) { + for (KvTablet tablet : currentKvs.values()) { + try { + if (tablet.canRelease(idleTimeoutMs, nowMs)) { + tablet.releaseKv(); + } + } catch (Exception e) { + LOG.warn("Failed to release idle KV tablet {}", tablet.getTableBucket(), e); + } + } } - /** Returns the maximum retry backoff after a failed lazy open. */ - public long getFailedBackoffMaxMs() { - return failedBackoffMaxMs; + /** Creates an unregistered tablet that opens RocksDB on first access. */ + public KvTablet createLazyTablet(PhysicalTablePath path, TableBucket bucket, LogTablet log) { + KvTablet tablet = + new KvTablet( + path, + bucket, + log, + getTabletDir(log.getDataDir(), path, bucket), + serverMetricGroup); + tablet.getLifecycle().configureTiming(clock, openSemaphore, RELEASE_DRAIN_TIMEOUT_MS); + return tablet; } /** @@ -604,26 +584,16 @@ public KvTablet getOrCreateKv( ArrowCompressionInfo arrowCompressionInfo, @Nullable Runnable flushCompleteListener) throws Exception { - return inKvLock( + return createKv( + tablePath, tableBucket, - () -> { - KvTablet existing = currentKvs.get(tableBucket); - if (existing != null) { - return existing; - } - KvTablet tablet = - doCreateKv( - tablePath, - tableBucket, - logTablet, - kvFormat, - schemaGetter, - tableConfig, - arrowCompressionInfo, - flushCompleteListener); - currentKvs.put(tableBucket, tablet); - return tablet; - }); + logTablet, + kvFormat, + schemaGetter, + tableConfig, + arrowCompressionInfo, + flushCompleteListener, + true); } /** Creates a tablet without replacing the registered lazy tablet. */ @@ -637,21 +607,19 @@ public KvTablet createKvTabletUnregistered( ArrowCompressionInfo arrowCompressionInfo, @Nullable Runnable flushCompleteListener) throws Exception { - return inKvLock( + return createKv( + tablePath, tableBucket, - () -> - doCreateKv( - tablePath, - tableBucket, - logTablet, - kvFormat, - schemaGetter, - tableConfig, - arrowCompressionInfo, - flushCompleteListener)); + logTablet, + kvFormat, + schemaGetter, + tableConfig, + arrowCompressionInfo, + flushCompleteListener, + false); } - private KvTablet doCreateKv( + private KvTablet createKv( PhysicalTablePath tablePath, TableBucket tableBucket, LogTablet logTablet, @@ -659,47 +627,60 @@ private KvTablet doCreateKv( SchemaGetter schemaGetter, TableConfig tableConfig, ArrowCompressionInfo arrowCompressionInfo, - @Nullable Runnable flushCompleteListener) + @Nullable Runnable flushCompleteListener, + boolean register) throws Exception { - File tabletDir = getOrCreateTabletDir(logTablet.getDataDir(), tablePath, tableBucket); - RowMerger merger = RowMerger.create(tableConfig, kvFormat, schemaGetter); - AutoIncrementManager autoIncrementManager = - new AutoIncrementManager( - schemaGetter, - tablePath.getTablePath(), - tableConfig, - new ZkSequenceGeneratorFactory(zkClient)); + return inKvLock( + tableBucket, + () -> { + if (register && currentKvs.containsKey(tableBucket)) { + return currentKvs.get(tableBucket); + } - KvTablet tablet = - KvTablet.create( - tablePath, - tableBucket, - logTablet, - tabletDir, - conf, - serverMetricGroup, - arrowBufferAllocator, - memorySegmentPool, - kvFormat, - merger, - arrowCompressionInfo, - schemaGetter, - tableConfig.getChangelogImage(), - sharedRocksDBRateLimiter, - sharedBlockCache, - sharedWriteBufferManager, - kvFlushScheduler, - flushCompleteListener, - autoIncrementManager, - clock, - tableConfig); + File tabletDir = + getOrCreateTabletDir(logTablet.getDataDir(), tablePath, tableBucket); + RowMerger merger = RowMerger.create(tableConfig, kvFormat, schemaGetter); + AutoIncrementManager autoIncrementManager = + new AutoIncrementManager( + schemaGetter, + tablePath.getTablePath(), + tableConfig, + new ZkSequenceGeneratorFactory(zkClient)); - LOG.info( - "Created kv tablet for bucket {} in dir {}.", - tableBucket, - tabletDir.getAbsolutePath()); + KvTablet tablet = + KvTablet.create( + tablePath, + tableBucket, + logTablet, + tabletDir, + conf, + serverMetricGroup, + arrowBufferAllocator, + memorySegmentPool, + kvFormat, + merger, + arrowCompressionInfo, + schemaGetter, + tableConfig.getChangelogImage(), + sharedRocksDBRateLimiter, + sharedBlockCache, + sharedWriteBufferManager, + kvFlushScheduler, + flushCompleteListener, + autoIncrementManager, + clock, + tableConfig); + if (register) { + currentKvs.put(tableBucket, tablet); + } - return tablet; + LOG.info( + "Created kv tablet for bucket {} in dir {}.", + tableBucket, + tabletDir.getAbsolutePath()); + + return tablet; + }); } /** @@ -903,20 +884,6 @@ public void registerKv(TableBucket tableBucket, KvTablet kvTablet) { }); } - /** Remove the expected tablet without unregistering a replacement for the same bucket. */ - public void unregisterKv(TableBucket tableBucket, KvTablet expectedTablet) { - inKvLock(tableBucket, () -> currentKvs.remove(tableBucket, expectedTablet)); - } - - /** - * Get the tablet directory path for the given table path and table bucket. This is a read-only - * accessor — it does not create the directory. - */ - public File getTabletDirPath( - File dataDir, PhysicalTablePath tablePath, TableBucket tableBucket) { - return getTabletDir(dataDir, tablePath, tableBucket); - } - public void deleteRemoteKvSnapshot( PhysicalTablePath physicalTablePath, TableBucket tableBucket) { FsPath remoteKvTabletDir = diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java index 627abb9adae..af8352b4226 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java @@ -137,31 +137,33 @@ public final class KvTablet { */ private static final int TARGET_ENTRIES_PER_NATIVE_WRITE = 500; - /** - * A guard that prevents the underlying RocksDB from being released while held. Use with - * try-with-resources. In eager mode, this is a no-op. In lazy mode, it pins the RocksDB - * instance and ensures it is open before returning. - */ + /** Pins an opened lazy tablet until closed; eager tablets use a no-op guard. */ public static final class Guard implements AutoCloseable { - private final @Nullable KvTablet tablet; + private final KvTablet tablet; + private final @Nullable KvTabletLazyLifecycle lifecycle; private boolean released; - /** No-op guard for eager mode. */ - static final Guard NOOP = new Guard(null); - - Guard(@Nullable KvTablet tablet) { + Guard(KvTablet tablet, @Nullable KvTabletLazyLifecycle lifecycle) { this.tablet = tablet; + this.lifecycle = lifecycle; + } + + /** Returns the opened tablet, valid until this guard is closed. */ + public KvTablet getTablet() { + return tablet; } @Override public void close() { - if (!released && tablet != null) { + if (!released && lifecycle != null) { released = true; - tablet.releasePin(); + lifecycle.releasePin(); } } } + private final Guard eagerGuard = new Guard(this, null); + private final PhysicalTablePath physicalPath; private final TableBucket tableBucket; private final boolean historicalPartition; @@ -301,10 +303,11 @@ private KvTablet( : 0L; } - private KvTablet( + KvTablet( PhysicalTablePath physicalPath, TableBucket tableBucket, LogTablet logTablet, + File kvTabletDir, @Nullable TabletServerMetricGroup serverMetricGroup) { this.physicalPath = physicalPath; this.tableBucket = tableBucket; @@ -312,7 +315,7 @@ private KvTablet( this.serverMetricGroup = serverMetricGroup; this.historicalPartition = HISTORICAL_PARTITION_VALUE.equals(physicalPath.getPartitionName()); - this.kvTabletDir = null; + this.kvTabletDir = kvTabletDir; this.writeBatchSize = 0; this.rocksDBKv = null; this.kvPreWriteBuffer = null; @@ -330,15 +333,6 @@ private KvTablet( this.lifecycle = new KvTabletLazyLifecycle(this); } - /** Creates a tablet whose RocksDB resources are opened on first access. */ - public static KvTablet createLazySentinel( - PhysicalTablePath physicalPath, - TableBucket tableBucket, - LogTablet logTablet, - @Nullable TabletServerMetricGroup serverMetricGroup) { - return new KvTablet(physicalPath, tableBucket, logTablet, serverMetricGroup); - } - /** Returns the lazy lifecycle, or null for an eagerly opened tablet. */ @Nullable public KvTabletLazyLifecycle getLifecycle() { @@ -355,10 +349,6 @@ void installRocksDB(KvTablet source) { } } - void detachRocksDB() { - detachRocksDB(KvCloseMode.PRESERVE_LOCAL_STATE); - } - void detachRocksDB(KvCloseMode closeMode) { KvTablet opened = openedTablet; if (opened != null) { @@ -371,7 +361,7 @@ void detachRocksDB(KvCloseMode closeMode) { } } - private KvTablet requireOpenedTablet() { + KvTablet requireOpenedTablet() { return checkNotNull(openedTablet, "RocksDB is not open for %s", tableBucket); } @@ -745,10 +735,6 @@ RowTtlTimestampProvider getRowTtlTimestampProvider() { /** Returns the physical value layout used by this tablet. */ public KvValueLayout getKvValueLayout() { - if (lifecycle != null) { - return requireOpenedTablet().getKvValueLayout(); - } - return kvValueLayout; } @@ -770,11 +756,6 @@ public long getAutoIncrementCacheSize() { } public void updateAutoIncrementIDRange(AutoIncIDRange newRange) { - if (lifecycle != null) { - requireOpenedTablet().updateAutoIncrementIDRange(newRange); - return; - } - autoIncrementManager.updateIDRange(newRange); } @@ -784,11 +765,6 @@ public String getPartitionName() { } public File getKvTabletDir() { - if (lifecycle != null) { - KvTablet opened = openedTablet; - return opened == null ? lifecycle.getTabletDir() : opened.getKvTabletDir(); - } - return kvTabletDir; } @@ -834,12 +810,6 @@ public long getRowCount() { if (opened != null) { return opened.getRowCount(); } - long cached = lifecycle.getCachedRowCount(); - if (cached == ROW_COUNT_DISABLED) { - throw new InvalidTableException( - "Row count is disabled for table " + getTablePath()); - } - return cached; } if (rowCount == ROW_COUNT_DISABLED) { @@ -918,9 +888,6 @@ public LogAppendInfo putAsLeader(KvRecordBatch kvRecords, @Nullable int[] target public LogAppendInfo putAsLeader( KvRecordBatch kvRecords, @Nullable int[] targetColumns, MergeMode mergeMode) throws Exception { - if (lifecycle != null) { - return requireOpenedTablet().putAsLeader(kvRecords, targetColumns, mergeMode); - } checkState( !historicalPartition, "putAsLeader is not supported for historical KV tablet %s", @@ -942,16 +909,6 @@ public LogAppendInfo putHistoricalAsLeader( String originalPartitionName, HistoricalValueLookup historicalValueLookup) throws Exception { - if (lifecycle != null) { - return requireOpenedTablet() - .putHistoricalAsLeader( - kvRecords, - targetColumns, - mergeMode, - originalPartitionName, - historicalValueLookup); - } - checkState(historicalPartition, "%s is not a historical KV tablet", tableBucket); return putAsLeader( kvRecords, @@ -973,12 +930,6 @@ public LocalValueLookupResult probeLocalPreviousValues( MergeMode mergeMode, String originalPartitionName) throws Exception { - if (lifecycle != null) { - return requireOpenedTablet() - .probeLocalPreviousValues( - kvRecords, targetColumns, mergeMode, originalPartitionName); - } - checkState(historicalPartition, "%s is not a historical KV tablet", tableBucket); return inReadLock( kvLock, @@ -1073,9 +1024,7 @@ public void setFlushCompleteListener(@Nullable Runnable flushCompleteListener) { public long getFlushedLogOffset() { if (lifecycle != null) { KvTablet opened = openedTablet; - return opened == null - ? lifecycle.getCachedFlushedLogOffset() - : opened.getFlushedLogOffset(); + return opened == null ? flushedLogOffset : opened.getFlushedLogOffset(); } return flushedLogOffset; @@ -1429,7 +1378,7 @@ public Executor getGuardedExecutor() { if (lifecycle != null) { KvTablet opened = openedTablet; return runnable -> { - Guard guard = tryAcquireExistingGuard(); + Guard guard = lifecycle.tryAcquireExistingGuard(); if (guard != null) { try (Guard ignored = guard) { if (opened != null && openedTablet == opened) { @@ -1444,10 +1393,6 @@ public Executor getGuardedExecutor() { } public List multiGet(List keys) throws IOException { - if (lifecycle != null) { - return requireOpenedTablet().multiGet(keys); - } - return inReadLock( kvLock, () -> { @@ -1465,10 +1410,6 @@ public List multiGet(List keys) throws IOException { * #multiGet} so that clients only observe flushed data. */ public List multiGetFromBufferOrKv(List keys) throws IOException { - if (lifecycle != null) { - return requireOpenedTablet().multiGetFromBufferOrKv(keys); - } - checkState( !historicalPartition, "multiGetFromBufferOrKv is not supported for historical KV tablet %s", @@ -1490,10 +1431,6 @@ public List multiGetFromBufferOrKv(List keys) throws IOE /** Looks up one key from the flushed local state for an original historical partition. */ public KvStateLookupResult lookupHistoricalLocal(String originalPartitionName, byte[] key) throws IOException { - if (lifecycle != null) { - return requireOpenedTablet().lookupHistoricalLocal(originalPartitionName, key); - } - checkState(historicalPartition, "%s is not a historical KV tablet", tableBucket); return inReadLock( kvLock, @@ -1512,10 +1449,6 @@ public KvStateLookupResult lookupHistoricalLocal(String originalPartitionName, b } public List prefixLookup(byte[] prefixKey) throws IOException { - if (lifecycle != null) { - return requireOpenedTablet().prefixLookup(prefixKey); - } - return inReadLock( kvLock, () -> { @@ -1525,10 +1458,6 @@ public List prefixLookup(byte[] prefixKey) throws IOException { } public List limitScan(int limit) throws IOException { - if (lifecycle != null) { - return requireOpenedTablet().limitScan(limit); - } - return inReadLock( kvLock, () -> { @@ -1556,10 +1485,6 @@ private List toValueBodySlices(List values) { */ public OpenScanResult openScan(String scannerId, long limit, long initialAccessTimeMs) throws IOException { - if (lifecycle != null) { - return requireOpenedTablet().openScan(scannerId, limit, initialAccessTimeMs); - } - return inReadLock( kvLock, () -> { @@ -1619,10 +1544,6 @@ public OpenScanResult openScan(String scannerId, long limit, long initialAccessT } public KvBatchWriter createKvBatchWriter() { - if (lifecycle != null) { - return requireOpenedTablet().createKvBatchWriter(); - } - return rocksDBKv.newWriteBatch( writeBatchSize, serverMetricGroup.kvFlushCount(), @@ -1642,7 +1563,7 @@ public void close() throws Exception { public void close(KvCloseMode closeMode) throws Exception { if (lifecycle != null) { - lifecycle.closeKvLazy(closeMode); + lifecycle.close(closeMode, false); return; } LOG.info( @@ -1676,7 +1597,7 @@ public void close(KvCloseMode closeMode) throws Exception { /** Completely delete the kv directory and all contents form the file system with no delay. */ public void drop() throws Exception { if (lifecycle != null) { - lifecycle.dropKvLazy(); + lifecycle.close(KvCloseMode.DISCARD_UNPERSISTED_STATE, true); return; } @@ -1834,38 +1755,11 @@ public boolean isLazyOpen() { return lifecycle != null && lifecycle.isOpen(); } - /** Returns the current lazy state. */ - public KvTabletLazyLifecycle.LazyState getLazyState() { - return lifecycle != null ? lifecycle.getLazyState() : null; - } - /** - * Acquire a guard that prevents RocksDB from being released while held. In eager mode, returns - * a no-op guard immediately. In lazy mode, ensures RocksDB is open (blocking if necessary) and - * pins it. - * - *

Must be called OUTSIDE any Replica-level locks (e.g. leaderIsrUpdateLock) to avoid - * blocking leader transitions during slow opens. + * Opens and pins this tablet. Call outside replica locks to avoid blocking leadership changes. */ public Guard acquireGuard() { - return lifecycle != null ? lifecycle.acquireGuard() : Guard.NOOP; - } - - /** - * Try to acquire a guard only if RocksDB is already open. - * - *

Returns {@code null} for a lazy tablet that is currently LAZY/FAILED/RELEASING/CLOSED. - */ - @Nullable - Guard tryAcquireExistingGuard() { - return lifecycle != null ? lifecycle.tryAcquireExistingGuard() : Guard.NOOP; - } - - /** Release a pin. Called by Guard.close(). */ - void releasePin() { - if (lifecycle != null) { - lifecycle.releasePin(); - } + return lifecycle != null ? lifecycle.acquireGuard() : eagerGuard; } /** Pre-check for idle release eligibility. */ @@ -1876,58 +1770,4 @@ public boolean canRelease(long closeIdleIntervalMs, long nowMs) { public boolean releaseKv() { return lifecycle != null && lifecycle.releaseKv(); } - - public void dropKvLazy() { - if (lifecycle != null) { - lifecycle.dropKvLazy(); - } - } - - // ---- Lazy state queries ---- - - public long getCachedRowCount() { - return lifecycle != null ? lifecycle.getCachedRowCount() : 0; - } - - public long getCachedFlushedLogOffset() { - return lifecycle != null ? lifecycle.getCachedFlushedLogOffset() : 0; - } - - public long getLastAccessTimestamp() { - return lifecycle != null ? lifecycle.getLastAccessTimestamp() : 0; - } - - /** - * Returns the number of active pins preventing RocksDB release. - * - *

This is primarily for testing and monitoring purposes. Exposing internal concurrency state - * may lead to misuse in production code. - */ - @VisibleForTesting - public int getActivePins() { - return lifecycle != null ? lifecycle.getActivePins() : 0; - } - - // ---- Test helpers ---- - - @VisibleForTesting - void setLazyStateForTesting(KvTabletLazyLifecycle.LazyState state) { - if (lifecycle != null) { - lifecycle.setLazyStateForTesting(state); - } - } - - @VisibleForTesting - void setLastAccessTimestampForTesting(long timestamp) { - if (lifecycle != null) { - lifecycle.setLastAccessTimestampForTesting(timestamp); - } - } - - @VisibleForTesting - void setClockForTesting(Clock clock) { - if (lifecycle != null) { - lifecycle.setClockForTesting(clock); - } - } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTabletLazyLifecycle.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTabletLazyLifecycle.java index c89ddd4d8ce..6bcad6ebebe 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTabletLazyLifecycle.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTabletLazyLifecycle.java @@ -17,11 +17,9 @@ package org.apache.fluss.server.kv; -import org.apache.fluss.annotation.VisibleForTesting; import org.apache.fluss.exception.KvStorageException; import org.apache.fluss.server.metrics.group.TabletServerMetricGroup; import org.apache.fluss.utils.ExponentialBackoff; -import org.apache.fluss.utils.FileUtils; import org.apache.fluss.utils.clock.Clock; import org.apache.fluss.utils.clock.SystemClock; @@ -31,23 +29,18 @@ import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; -import java.io.File; import java.util.concurrent.CancellationException; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; import java.util.function.IntSupplier; -import java.util.function.Supplier; /** - * Manages the lazy lifecycle state machine for a {@link KvTablet}. Encapsulates all state - * transitions (LAZY → OPENING → OPEN → RELEASING → LAZY), pin management, open/release/drop logic, - * and callback orchestration. - * - *

This class is only instantiated for lazy-mode tablets. EAGER-mode tablets do not use this - * class. + * Opens KV resources on demand and releases idle tablets. Close/drop fence new requests and wait + * for pins, callbacks and candidate cleanup before relinquishing native resources or local paths. */ public final class KvTabletLazyLifecycle { private static final Logger LOG = LoggerFactory.getLogger(KvTabletLazyLifecycle.class); @@ -55,74 +48,36 @@ public final class KvTabletLazyLifecycle { /** Coarsen access timestamp updates to reduce volatile writes on hot path. */ private static final long ACCESS_TIMESTAMP_GRANULARITY_MS = 1000; - // ---- Lazy lifecycle states ---- + private static final long OPEN_TIMEOUT_MS = 300_000; /** Internal RocksDB lazy lifecycle states. */ enum LazyState { - /** Lazy mode: RocksDB not yet opened. */ LAZY, - /** Lazy mode: RocksDB is being opened by another thread. */ + OPENING, - /** Lazy mode: RocksDB is open and serving requests. */ + OPEN, - /** Lazy mode: RocksDB is being released (closing without deleting data). */ + RELEASING, - /** Lazy mode: last open attempt failed, in backoff cooldown. */ + FAILED, /** Closing: new requests are fenced while outstanding work drains. */ CLOSING, - /** - * Terminal state: tablet is closed and will not be reopened. Local data may or may not have - * been deleted depending on whether close or drop was called. - */ + /** Terminal state; local data is deleted only on drop. */ CLOSED } - // ---- Callback interfaces ---- - - /** - * Callback for the actual RocksDB open work. Provided by Replica since it requires access to - * snapshot context, log recovery, etc. - */ + /** Creates an eager tablet, recovering local SST/WAL or a snapshot as needed. */ @FunctionalInterface public interface OpenCallback { /** Performs the open and returns the opened RocksDB KvTablet instance. */ KvTablet doOpen(boolean hasLocalData) throws Exception; } - /** - * Callback invoked after a successful open commit. Used by Replica to install the tablet - * reference and start periodic snapshot. - */ - @FunctionalInterface - public interface OpenCommitCallback { - /** Called after RocksDB state has been installed into the sentinel. */ - void onOpenCommitted(KvTablet kvTablet); - } - - /** Callback for releasing (closing) RocksDB without destroying local data. */ - @FunctionalInterface - public interface ReleaseCallback { - /** Performs release-time cleanup (e.g. stop snapshots, unregister metrics). */ - void doRelease(KvTablet kvTablet); - } - - /** Callback for dropping (destroying) a KvTablet and cleaning up associated resources. */ - @FunctionalInterface - public interface DropCallback { - /** Performs drop-time cleanup (e.g. unregister metrics, stop snapshots). */ - void doDrop(KvTablet kvTablet); - } - - // ---- The owning tablet ---- - private final KvTablet tablet; - // ---- State machine fields ---- - private volatile LazyState lazyState = LazyState.LAZY; - /** Lock guarding lazy state transitions. */ private final ReentrantLock lazyStateLock = new ReentrantLock(); private final Condition lazyStateChanged = lazyStateLock.newCondition(); @@ -130,30 +85,18 @@ public interface DropCallback { /** Active pin count — prevents release while operations are in-flight. */ private final AtomicInteger activePins = new AtomicInteger(0); - private volatile boolean rejectNewPins; - - /** Generation counter for fencing stale open results. */ - private long openGeneration; - /** Semaphore for throttling concurrent open operations across all tablets. */ private @Nullable Semaphore openSemaphore; - private long openTimeoutMs; private long releaseDrainTimeoutMs; - private @Nullable ExponentialBackoff failedBackoff; + private final ExponentialBackoff failedBackoff = new ExponentialBackoff(5_000, 2, 300_000, 0); private Clock clock = SystemClock.getInstance(); - /** FAILED state tracking. */ private long failedTimestamp; private int failureCount; private @Nullable Throwable lastFailureCause; - /** Cached values served when RocksDB is not open. */ - private volatile long cachedRowCount; - - private volatile long cachedFlushedLogOffset; - /** Whether local RocksDB data directory exists from a previous open. */ @GuardedBy("lazyStateLock") private boolean hasLocalData; @@ -161,20 +104,14 @@ public interface DropCallback { /** Access timestamp for idle release decisions (coarsened to reduce volatile writes). */ private volatile long lastAccessTimestamp; - /** Epoch suppliers for fencing stale open results. */ private @Nullable IntSupplier leaderEpochSupplier; private @Nullable IntSupplier bucketEpochSupplier; - /** Supplier for tablet directory path — used by sentinel in LAZY/FAILED state. */ - private @Nullable Supplier tabletDirSupplier; - - /** Callbacks (set by Replica after construction). */ private @Nullable OpenCallback openCallback; - private @Nullable OpenCommitCallback commitCallback; - private @Nullable DropCallback dropCallback; - private @Nullable ReleaseCallback releaseCallback; + private @Nullable Consumer commitCallback; + private @Nullable Consumer releaseCallback; /** * Includes open callbacks, candidate cleanup and idle release cleanup. Guarded by the state @@ -190,270 +127,95 @@ public interface DropCallback { KvTabletLazyLifecycle(KvTablet tablet) { this.tablet = tablet; + updateStateGauge(LazyState.LAZY, 1); } - // ---- Configuration (called by Replica after sentinel creation) ---- - - /** - * Configure lazy open parameters. Must be called before any {@link #acquireGuard()} calls. - * - * @param clock the clock for time-based operations - * @param openSemaphore semaphore to limit concurrent open operations (can be null) - * @param openTimeoutMs timeout for open operations in milliseconds - * @param failedBackoffBaseMs base backoff time for failed opens in milliseconds - * @param failedBackoffMaxMs max backoff time for failed opens in milliseconds - * @param releaseDrainTimeoutMs timeout for draining pins during release in milliseconds - */ - public void configureLazyOpen( - Clock clock, - Semaphore openSemaphore, - long openTimeoutMs, - long failedBackoffBaseMs, - long failedBackoffMaxMs, - long releaseDrainTimeoutMs) { - this.clock = clock; - this.openSemaphore = openSemaphore; - this.openTimeoutMs = openTimeoutMs; - this.failedBackoff = new ExponentialBackoff(failedBackoffBaseMs, 2, failedBackoffMaxMs, 0); - this.releaseDrainTimeoutMs = releaseDrainTimeoutMs; - // Register initial LAZY state in metrics - updateStateGauges(null, LazyState.LAZY); - } - - /** - * Sets the callback responsible for performing the actual RocksDB open operation. This is - * provided by Replica since it requires access to snapshot context, log recovery, etc. - * - * @param callback the open callback to invoke when transitioning from LAZY to OPEN - */ - public void setOpenCallback(OpenCallback callback) { - this.openCallback = callback; + /** Configures recovery before this sentinel is registered or accessed. */ + public void configure( + IntSupplier leaderEpoch, + IntSupplier bucketEpoch, + OpenCallback open, + Consumer committed, + Consumer release) { + leaderEpochSupplier = leaderEpoch; + bucketEpochSupplier = bucketEpoch; + openCallback = open; + commitCallback = committed; + releaseCallback = release; } - /** - * Sets the callback invoked after a successful open commit. Used by Replica to install the - * tablet reference and start periodic snapshot. - * - * @param callback the commit callback to invoke after RocksDB state is installed - */ - public void setCommitCallback(OpenCommitCallback callback) { - this.commitCallback = callback; - } - - /** - * Sets the callback for full-state cleanup when the tablet is dropped. Handles all states, - * waits for in-progress operations, then transitions to CLOSED. - * - * @param callback the drop callback to invoke during tablet deletion - */ - public void setDropCallback(DropCallback callback) { - this.dropCallback = callback; - } - - /** - * Sets the callback for releasing (closing) RocksDB without destroying local data. Called by - * the idle release controller when transitioning from OPEN to LAZY. - * - * @param callback the release callback to invoke during idle release - */ - public void setReleaseCallback(ReleaseCallback callback) { - this.releaseCallback = callback; - } - - /** - * Sets the supplier for the tablet directory path. Used by sentinel in LAZY/FAILED state for - * cleanup operations. - * - * @param supplier the directory supplier - */ - public void setTabletDirSupplier(Supplier supplier) { - this.tabletDirSupplier = supplier; - } - - /** Returns the tablet directory from the supplier, or null if not configured. */ - @Nullable - File getTabletDir() { - return tabletDirSupplier != null ? tabletDirSupplier.get() : null; - } - - /** - * Sets the supplier for the leader epoch used in fencing stale open operations. - * - * @param supplier the leader epoch supplier - */ - public void setLeaderEpochSupplier(IntSupplier supplier) { - this.leaderEpochSupplier = supplier; - } - - /** - * Sets the supplier for the bucket epoch used in fencing stale open operations. - * - * @param supplier the bucket epoch supplier - */ - public void setBucketEpochSupplier(IntSupplier supplier) { - this.bucketEpochSupplier = supplier; + void configureTiming(Clock clock, Semaphore semaphore, long drainTimeout) { + this.clock = clock; + this.openSemaphore = semaphore; + this.releaseDrainTimeoutMs = drainTimeout; } - /** - * Initializes the cached row count from snapshot metadata. Called during sentinel creation to - * serve read queries while in LAZY state before the first open. Also sets the sentinel's - * rowCount field so that OPENING state reads return the correct value. - * - * @param rowCount the initial row count (-1 if unknown/disabled) - */ + /** Initializes row count from snapshot metadata without opening RocksDB. */ public void initCachedRowCount(long rowCount) { - this.cachedRowCount = rowCount; tablet.setRowCount(rowCount); } - // ---- State queries ---- - - /** Returns whether the tablet is currently in OPEN state with RocksDB loaded and serving. */ boolean isOpen() { return lazyState == LazyState.OPEN; } - /** Returns the current lifecycle state of this lazy tablet. */ LazyState getLazyState() { return lazyState; } - /** - * Returns the cached row count from the last flush or release operation. This value may be - * stale but is sufficient for serving read queries while in LAZY state. - */ - long getCachedRowCount() { - return cachedRowCount; - } - - /** - * Returns the flushed log offset cached from the last flush or release operation. Used to - * determine how much incremental log needs to be replayed when reopening from local data. - */ - long getCachedFlushedLogOffset() { - return cachedFlushedLogOffset; + /** Returns the row count retained across idle release. */ + public long getCachedRowCount() { + return tablet.currentRowCount(); } - /** - * Returns the timestamp (in milliseconds) of the last access to this tablet. Used by the idle - * release controller to determine eligibility for release. - */ long getLastAccessTimestamp() { return lastAccessTimestamp; } - /** - * Returns the number of active pins that are currently preventing RocksDB from being released. - * Each pin corresponds to a {@link KvTablet.Guard} held by an in-flight operation. - */ int getActivePins() { return activePins.get(); } /** - * Returns true if this tablet is in LAZY or FAILED state (RocksDB not open), meaning - * KvTablet.getRowCount() should return the cached value instead of querying RocksDB. - */ - boolean needsCachedRowCount() { - LazyState s = lazyState; - return s == LazyState.LAZY || s == LazyState.FAILED; - } - - // ---- Guard / Pin management ---- - - /** - * Acquire a guard that prevents RocksDB from being released while held. Ensures RocksDB is open - * (blocking if necessary) and pins it. - * - *

Must be called OUTSIDE any Replica-level locks (e.g. leaderIsrUpdateLock) to avoid - * blocking leader transitions during slow opens. - * - * @throws IllegalStateException if the lifecycle has not been fully configured via {@link - * #configureLazyOpen} and callback setters + * Opens and pins the tablet. Call outside replica locks to avoid blocking leadership changes. */ KvTablet.Guard acquireGuard() { - // Defensive check: ensure all required callbacks are set before allowing access if (openCallback == null) { - throw new IllegalStateException( - "KvTabletLazyLifecycle not fully configured: openCallback is null. " - + "Ensure configureLazyOpen() and setOpenCallback() are called before use."); + throw new IllegalStateException("Lazy tablet recovery is not configured."); } - // Fast path: try to pin without blocking - KvTablet.Guard fast = tryAcquirePinAsGuard(); - if (fast != null) { - touchAccessTimestamp(); - return fast; - } - - // Slow path: ensure open, then pin - ensureOpen(); - // Between ensureOpen() returning and pin, a concurrent release could happen. - // Retry up to 3 times. for (int attempt = 0; ; attempt++) { - lazyStateLock.lock(); - try { - if (lazyState == LazyState.OPEN && !rejectNewPins) { - activePins.incrementAndGet(); - touchAccessTimestamp(); - return new KvTablet.Guard(tablet); - } - if (lazyState == LazyState.CLOSED || lazyState == LazyState.CLOSING) { - throw new KvStorageException( - "KvTablet is closed for " + tablet.getTableBucket()); - } - } finally { - lazyStateLock.unlock(); + KvTablet.Guard guard = tryAcquireExistingGuard(); + if (guard != null) { + touchAccessTimestamp(); + return guard; } - if (attempt >= 2) { - throw new KvStorageException( - "Failed to pin KvTablet after open for " + tablet.getTableBucket()); + if (attempt >= 3) { + throw new KvStorageException("Failed to pin KV tablet " + tablet.getTableBucket()); } - // Re-open and retry ensureOpen(); } } /** - * Try to acquire a guard only if RocksDB is already OPEN. - * - *

Unlike {@link #acquireGuard()}, this method never triggers a lazy open. It is intended for - * background maintenance paths (flush, snapshot init) that must coordinate with idle release - * but should not reopen a lazy tablet on their own. + * Pins maintenance without refreshing idle time. Recheck after incrementing so release cannot + * miss a newly admitted request. */ @Nullable KvTablet.Guard tryAcquireExistingGuard() { - return tryAcquirePinAsGuard(); - } - - /** - * Fast-path pin attempt. Returns Guard if OPEN and accepting pins; null otherwise. - * - *

Safety relies on the double-check pattern: we optimistically increment the pin count, then - * re-verify the state. If the state changed concurrently (e.g., release started), the re-check - * fails and we undo the increment. This is safe on all architectures (x86, ARM) regardless of - * volatile read ordering between independent variables. - */ - @Nullable - private KvTablet.Guard tryAcquirePinAsGuard() { - if (lazyState == LazyState.OPEN && !rejectNewPins) { + if (lazyState == LazyState.OPEN) { activePins.incrementAndGet(); - if (lazyState == LazyState.OPEN && !rejectNewPins) { - return new KvTablet.Guard(tablet); + if (lazyState == LazyState.OPEN) { + return new KvTablet.Guard(tablet.requireOpenedTablet(), this); } - decrementAndMaybeSignal(); + releasePin(); } return null; } - /** Release a pin. Called by Guard.close(). */ void releasePin() { - decrementAndMaybeSignal(); - } - - private void decrementAndMaybeSignal() { if (activePins.decrementAndGet() == 0) { - if (rejectNewPins) { + if (lazyState != LazyState.OPEN) { lazyStateLock.lock(); try { lazyStateChanged.signalAll(); @@ -471,51 +233,22 @@ private void touchAccessTimestamp() { } } - /** - * Update state gauge metrics on state transitions. Only LAZY, OPEN, FAILED are tracked as - * "stable" states; OPENING, RELEASING, CLOSED fall into default (no-op). - */ - private void updateStateGauges(@Nullable LazyState from, LazyState to) { - TabletServerMetricGroup metricGroup = tablet.serverMetricGroup; - if (metricGroup == null) { + /** Counts stable states only; transitional states have no gauge. */ + private void updateStateGauge(LazyState state, int delta) { + TabletServerMetricGroup metrics = tablet.serverMetricGroup; + if (metrics == null) { return; } - if (from != null) { - switch (from) { - case LAZY: - metricGroup.kvTabletLazyCount().decrementAndGet(); - break; - case OPEN: - metricGroup.kvTabletOpenCount().decrementAndGet(); - break; - case FAILED: - metricGroup.kvTabletFailedCount().decrementAndGet(); - break; - default: - break; - } - } - switch (to) { - case LAZY: - metricGroup.kvTabletLazyCount().incrementAndGet(); - break; - case OPEN: - metricGroup.kvTabletOpenCount().incrementAndGet(); - break; - case FAILED: - metricGroup.kvTabletFailedCount().incrementAndGet(); - break; - default: - break; + if (state == LazyState.LAZY) { + metrics.kvTabletLazyCount().addAndGet(delta); + } else if (state == LazyState.OPEN) { + metrics.kvTabletOpenCount().addAndGet(delta); + } else if (state == LazyState.FAILED) { + metrics.kvTabletFailedCount().addAndGet(delta); } } - // ---- Open management ---- - - /** - * Ensures RocksDB is open. If already OPEN, returns immediately. If LAZY or FAILED (past - * cooldown), triggers a slow-path open. If OPENING or RELEASING, blocks until state changes. - */ + /** Waits for ongoing work or starts a single open after failure backoff. */ private void ensureOpen() { lazyStateLock.lock(); try { @@ -525,17 +258,10 @@ private void ensureOpen() { return; case OPENING: - if (!lazyStateChanged.await(openTimeoutMs, TimeUnit.MILLISECONDS)) { - throw new KvStorageException( - "KvTablet open timed out for " + tablet.getTableBucket()); - } - continue; - case RELEASING: - if (!lazyStateChanged.await(openTimeoutMs, TimeUnit.MILLISECONDS)) { + if (!lazyStateChanged.await(OPEN_TIMEOUT_MS, TimeUnit.MILLISECONDS)) { throw new KvStorageException( - "KvTablet open timed out waiting for release: " - + tablet.getTableBucket()); + "Timed out opening KV tablet " + tablet.getTableBucket()); } continue; @@ -556,16 +282,13 @@ private void ensureOpen() { case LAZY: transitionTo(LazyState.OPENING); operationInFlight = true; - openGeneration++; - long myGeneration = openGeneration; - int myLeaderEpoch = - leaderEpochSupplier != null ? leaderEpochSupplier.getAsInt() : -1; - int myBucketEpoch = - bucketEpochSupplier != null ? bucketEpochSupplier.getAsInt() : -1; + + int myLeaderEpoch = leaderEpochSupplier.getAsInt(); + int myBucketEpoch = bucketEpochSupplier.getAsInt(); boolean localDataExists = hasLocalData; lazyStateLock.unlock(); try { - doSlowOpen(myGeneration, myLeaderEpoch, myBucketEpoch, localDataExists); + doSlowOpen(myLeaderEpoch, myBucketEpoch, localDataExists); } catch (Throwable t) { // doSlowOpen completes candidate cleanup before publishing failure. if (t instanceof RuntimeException) { @@ -596,8 +319,7 @@ private void ensureOpen() { } } - private void doSlowOpen( - long myGeneration, int myLeaderEpoch, int myBucketEpoch, boolean localDataExists) + private void doSlowOpen(int myLeaderEpoch, int myBucketEpoch, boolean localDataExists) throws Exception { KvTablet candidate = null; boolean committed = false; @@ -606,7 +328,8 @@ private void doSlowOpen( boolean cleanupFailed = false; try { if (openSemaphore != null) { - semaphoreAcquired = openSemaphore.tryAcquire(openTimeoutMs, TimeUnit.MILLISECONDS); + semaphoreAcquired = + openSemaphore.tryAcquire(OPEN_TIMEOUT_MS, TimeUnit.MILLISECONDS); if (!semaphoreAcquired) { throw new KvStorageException( "KvTablet open semaphore timeout for " + tablet.getTableBucket()); @@ -615,29 +338,26 @@ private void doSlowOpen( candidate = openCallback.doOpen(localDataExists); lazyStateLock.lock(); try { - int leaderEpoch = leaderEpochSupplier != null ? leaderEpochSupplier.getAsInt() : -1; - int bucketEpoch = bucketEpochSupplier != null ? bucketEpochSupplier.getAsInt() : -1; + int leaderEpoch = leaderEpochSupplier.getAsInt(); + int bucketEpoch = bucketEpochSupplier.getAsInt(); if (lazyState != LazyState.OPENING - || openGeneration != myGeneration || leaderEpoch != myLeaderEpoch || bucketEpoch != myBucketEpoch) { throw new CancellationException( - "open result fenced by concurrent epoch/generation change"); + "open result fenced by concurrent epoch or lifecycle change"); } tablet.installRocksDB(candidate); committed = true; transitionTo(LazyState.OPEN); failureCount = 0; lastAccessTimestamp = clock.milliseconds(); - rejectNewPins = false; + lazyStateChanged.signalAll(); } finally { lazyStateLock.unlock(); } try { - if (commitCallback != null) { - commitCallback.onOpenCommitted(tablet); - } + commitCallback.accept(tablet); } catch (Exception e) { LOG.warn("Post-open commit callback failed for {}", tablet.getTableBucket(), e); } @@ -648,10 +368,7 @@ private void doSlowOpen( try { if (!committed && candidate != null) { candidate.close(); - File dir = candidate.getKvTabletDir(); - if (dir != null) { - FileUtils.deleteDirectory(dir); - } + candidate.deleteLocalDirectory(); } } catch (Exception | Error e) { cleanupFailed = true; @@ -669,10 +386,14 @@ private void doSlowOpen( try { if (cleanupFailed) { failedOpenTablet = candidate; - rejectNewPins = true; + transitionTo(LazyState.CLOSING); } else if (!committed && lazyState != LazyState.CLOSING) { - transitionToFailed(failure, failure instanceof CancellationException); + transitionTo(LazyState.FAILED); + failedTimestamp = clock.milliseconds(); + failureCount = + failure instanceof CancellationException ? 0 : failureCount + 1; + lastFailureCause = failure; } operationInFlight = false; lazyStateChanged.signalAll(); @@ -685,41 +406,19 @@ private void doSlowOpen( /** Must be called while holding {@code lazyStateLock}. */ private void transitionTo(LazyState state) { - updateStateGauges(lazyState, state); + updateStateGauge(lazyState, -1); + updateStateGauge(state, 1); lazyState = state; } - /** Must be called while holding {@code lazyStateLock}. */ - private void transitionToFailed(Throwable cause, boolean resetCount) { - transitionTo(LazyState.FAILED); - failedTimestamp = clock.milliseconds(); - if (resetCount) { - failureCount = 0; - } else { - failureCount++; - } - lastFailureCause = cause; - lazyStateChanged.signalAll(); - } - - // ---- Release logic (idle release by KvManager) ---- - - /** Pre-check for idle release eligibility. */ boolean canRelease(long closeIdleIntervalMs, long nowMs) { - if (lazyState != LazyState.OPEN) { - return false; - } - if (activePins.get() > 0) { - return false; - } - return nowMs - lastAccessTimestamp >= closeIdleIntervalMs; + return lazyState == LazyState.OPEN + && activePins.get() == 0 + && nowMs - lastAccessTimestamp >= closeIdleIntervalMs; } /** - * Release RocksDB resources: OPEN -> RELEASING -> LAZY. Caches metadata before close for - * serving queries while in LAZY state. - * - * @return true if release succeeded, false if aborted + * Releases idle native resources; returns false if data, leases or callbacks prevent release. */ boolean releaseKv() { lazyStateLock.lock(); @@ -727,7 +426,7 @@ boolean releaseKv() { if (lazyState != LazyState.OPEN || operationInFlight) { return false; } - rejectNewPins = true; + transitionTo(LazyState.RELEASING); operationInFlight = true; if (!drainPins() @@ -735,14 +434,14 @@ boolean releaseKv() { || tablet.getFlushedLogOffset() < tablet.logTablet.localLogEndOffset() || tablet.hasActiveResourceLeases()) { if (lazyState == LazyState.RELEASING) { - transitionBackToOpen(); + transitionTo(LazyState.OPEN); } operationInFlight = false; lazyStateChanged.signalAll(); return false; } - cachedRowCount = tablet.currentRowCount(); - cachedFlushedLogOffset = tablet.getFlushedLogOffset(); + tablet.setRowCount(tablet.currentRowCount()); + tablet.setFlushedLogOffset(tablet.getFlushedLogOffset()); } finally { lazyStateLock.unlock(); } @@ -751,10 +450,10 @@ boolean releaseKv() { boolean nativeCloseStarted = false; try { if (releaseCallback != null) { - releaseCallback.doRelease(tablet); + releaseCallback.accept(tablet); } nativeCloseStarted = true; - tablet.detachRocksDB(); + tablet.detachRocksDB(KvCloseMode.PRESERVE_LOCAL_STATE); releaseSucceeded = true; } catch (Exception e) { LOG.warn("Failed to release KV tablet {}", tablet.getTableBucket(), e); @@ -765,9 +464,9 @@ boolean releaseKv() { if (releaseSucceeded) { transitionTo(LazyState.LAZY); hasLocalData = true; - rejectNewPins = false; + } else if (!nativeCloseStarted) { - transitionBackToOpen(); + transitionTo(LazyState.OPEN); } else { transitionTo(LazyState.CLOSING); } @@ -781,22 +480,10 @@ boolean releaseKv() { return releaseSucceeded; } - /** Close resources and remove local data only after outstanding operations finish. */ - void dropKvLazy() { - terminate(KvCloseMode.DISCARD_UNPERSISTED_STATE, true); - } - - void closeKvLazy(KvCloseMode closeMode) { - terminate(closeMode, false); - } - /** Serializes close callers without holding the state lock during callbacks or native I/O. */ - private synchronized void terminate(KvCloseMode closeMode, boolean deleteDirectory) { + synchronized void close(KvCloseMode closeMode, boolean deleteDirectory) { if (lazyState == LazyState.CLOSED) { if (deleteDirectory && !localDirectoryDeleted) { - if (dropCallback != null) { - dropCallback.doDrop(tablet); - } tablet.deleteLocalDirectory(); localDirectoryDeleted = true; } @@ -804,8 +491,7 @@ private synchronized void terminate(KvCloseMode closeMode, boolean deleteDirecto } lazyStateLock.lock(); try { - rejectNewPins = true; - openGeneration++; + transitionTo(LazyState.CLOSING); lazyStateChanged.signalAll(); // A request timeout cannot transfer ownership of native resources or the local path. @@ -816,20 +502,13 @@ private synchronized void terminate(KvCloseMode closeMode, boolean deleteDirecto lazyStateLock.unlock(); } - if (deleteDirectory) { - if (dropCallback != null) { - dropCallback.doDrop(tablet); - } - } else if (releaseCallback != null) { - releaseCallback.doRelease(tablet); + if (releaseCallback != null) { + releaseCallback.accept(tablet); } if (failedOpenTablet != null) { try { failedOpenTablet.close(); - File dir = failedOpenTablet.getKvTabletDir(); - if (dir != null) { - FileUtils.deleteDirectory(dir); - } + failedOpenTablet.deleteLocalDirectory(); failedOpenTablet = null; } catch (Exception e) { throw new KvStorageException( @@ -844,9 +523,6 @@ private synchronized void terminate(KvCloseMode closeMode, boolean deleteDirecto lazyStateLock.lock(); try { transitionTo(LazyState.CLOSED); - hasLocalData = false; - failureCount = 0; - failedTimestamp = 0; lastFailureCause = null; lazyStateChanged.signalAll(); } finally { @@ -870,34 +546,4 @@ private boolean drainPins() { } return true; } - - /** Must be called while holding {@code lazyStateLock}. */ - private void transitionBackToOpen() { - transitionTo(LazyState.OPEN); - rejectNewPins = false; - lazyStateChanged.signalAll(); - } - - // ---- Test helpers ---- - - @VisibleForTesting - void setLazyStateForTesting(LazyState state) { - lazyStateLock.lock(); - try { - this.lazyState = state; - lazyStateChanged.signalAll(); - } finally { - lazyStateLock.unlock(); - } - } - - @VisibleForTesting - void setLastAccessTimestampForTesting(long timestamp) { - this.lastAccessTimestamp = timestamp; - } - - @VisibleForTesting - void setClockForTesting(Clock clock) { - this.clock = clock; - } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index 5b8e1edb8c5..826186216b7 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -838,18 +838,10 @@ private void dropKv() { } KvTablet kv = this.kvTablet; if (kv != null) { - if (kv.isLazyMode()) { - // Delegate to KvTablet's lazy lifecycle which handles all states - // (dropCallback inside dropKvLazy already calls stopSnapshotManager) - kv.dropKvLazy(); - // Remove sentinel from KvManager registry - checkNotNull(kvManager); - kvManager.unregisterKv(tableBucket, kv); - } else { + if (!kv.isLazyMode()) { bucketMetricGroup.unregisterRocksDBStatistics(); - checkNotNull(kvManager); - kvManager.dropKv(tableBucket); } + checkNotNull(kvManager).dropKv(tableBucket); this.kvTablet = null; } } @@ -887,7 +879,7 @@ private void onKvFlushComplete() { * @return the snapshot used to init kv tablet, empty if no any snapshot. */ private Optional initKvTabletEager() { - InitKvResult result = initKvTablet(); + InitKvResult result = initKvTablet(null); kvManager.registerKv(tableBucket, result.tablet); kvTablet = result.tablet; if (kvTablet.getRocksDBStatistics() != null) { @@ -898,50 +890,17 @@ private Optional initKvTabletEager() { private void createKvLazy() { TableConfig tableConfig = getTableConfig(); - TabletServerMetricGroup serverMetrics = - bucketMetricGroup.getTableMetricGroup().getServerMetricGroup(); - - // Create a lazy sentinel — no RocksDB yet - KvTablet sentinel = - KvTablet.createLazySentinel(physicalPath, tableBucket, logTablet, serverMetrics); - - // Configure lazy open parameters via lifecycle + KvTablet sentinel = kvManager.createLazyTablet(physicalPath, tableBucket, logTablet); KvTabletLazyLifecycle lifecycle = sentinel.getLifecycle(); - lifecycle.configureLazyOpen( - clock, - kvManager.getOpenSemaphore(), - kvManager.getOpenTimeoutMs(), - kvManager.getFailedBackoffBaseMs(), - kvManager.getFailedBackoffMaxMs(), - KvManager.RELEASE_DRAIN_TIMEOUT_MS); - lifecycle.setLeaderEpochSupplier(() -> leaderEpoch); - lifecycle.setBucketEpochSupplier(() -> bucketEpoch); - - // Tablet directory supplier for LAZY/FAILED state cleanup - lifecycle.setTabletDirSupplier( - () -> - kvManager.getTabletDirPath( - logTablet.getDataDir(), physicalPath, tableBucket)); - - // Shared state between open and commit callbacks to pass snapshot reference AtomicReference snapshotRef = new AtomicReference<>(); - - // Open callback: creates a real KvTablet - // If local data exists (reopen after idle release), load from local directory - // and replay only incremental logs. Otherwise, do full init (download snapshot). - lifecycle.setOpenCallback( + lifecycle.configure( + () -> leaderEpoch, + () -> bucketEpoch, hasLocal -> { - if (hasLocal) { - return reopenFromLocalData(sentinel, snapshotRef); - } else { - InitKvResult result = initKvTablet(); - snapshotRef.set(result.snapshotUsed.orElse(null)); - return result.tablet; - } - }); - - // Commit callback: start periodic snapshot after RocksDB is installed - lifecycle.setCommitCallback( + InitKvResult result = initKvTablet(hasLocal ? sentinel : null); + snapshotRef.set(result.snapshotUsed.orElse(null)); + return result.tablet; + }, kv -> { if (!isHistoricalPartition()) { startPeriodicKvSnapshot(kv, snapshotRef.getAndSet(null)); @@ -949,36 +908,10 @@ private void createKvLazy() { if (kv.getRocksDBStatistics() != null) { bucketMetricGroup.registerRocksDBStatistics(kv.getRocksDBStatistics()); } - }); - - // Release callback: stop snapshot and unregister metrics - // RocksDB close is handled by detachRocksDB() inside KvTablet - lifecycle.setReleaseCallback( - kv -> { - try { - stopSnapshotManager(); - } catch (Exception e) { - LOG.warn( - "Failed to stop kv snapshot manager during release for {}", - tableBucket, - e); - } - try { - bucketMetricGroup.unregisterRocksDBStatistics(); - } catch (Exception e) { - LOG.warn( - "Failed to unregister RocksDB statistics during release for {}", - tableBucket, - e); - } - }); - - // Drop callback: cleanup metrics and snapshot - // Registry removal is handled by Replica.dropKv() - lifecycle.setDropCallback( + }, kv -> { - bucketMetricGroup.unregisterRocksDBStatistics(); stopSnapshotManager(); + bucketMetricGroup.unregisterRocksDBStatistics(); }); // Initialize cached row count from snapshot metadata if available (no RocksDB needed). @@ -1010,37 +943,40 @@ private void createKvLazy() { this.kvTablet = sentinel; } - private InitKvResult initKvTablet() { + private InitKvResult initKvTablet(@Nullable KvTablet released) { checkNotNull(kvManager); TableConfig tableConfig = getTableConfig(); long startTime = clock.milliseconds(); LOG.info("Start to init kv tablet for {} of table {}.", tableBucket, physicalPath); - // todo: we may need to handle the following cases: - // case1: no kv files in local, restore from remote snapshot; and apply - // the log; - // case2: kv files in local - // - if no remote snapshot, restore from local and apply the log known to the local - // files. - // - have snapshot, if the known offset to the local files is much less than(maybe - // some value configured) - // the remote snapshot; restore from remote snapshot; - - // currently for simplicity, we'll always download the snapshot files and restore from - // the snapshots as kv files won't exist in our current implementation for - // when replica become follower, we'll always delete the kv files. - + // Released tablets retain local SSTs and the corresponding WAL replay position. + // Initial opens and failed local recovery use the latest durable snapshot instead. // get the offset from which, we should restore from. default is 0 - long restoreStartOffset = isHistoricalPartition() ? historicalRecoveryStartOffset() : 0; + long restoreStartOffset = + released == null && isHistoricalPartition() ? historicalRecoveryStartOffset() : 0; // Lake is the durable base for local historical KV state. Historical replicas therefore // never restore a normal KV snapshot, even if one exists from older code. Optional optCompletedSnapshot = - isHistoricalPartition() ? Optional.empty() : getLatestSnapshot(tableBucket); - KvTablet createdTablet = null; + released != null || isHistoricalPartition() + ? Optional.empty() + : getLatestSnapshot(tableBucket); + KvTablet kvTablet = null; try { Long rowCount; AutoIncIDRange autoIncIDRange; - if (optCompletedSnapshot.isPresent()) { + if (released != null) { + File directory = released.getKvTabletDir(); + if (!new File(directory, RocksDBKvBuilder.DB_INSTANCE_DIR_STRING).isDirectory()) { + throw new IOException("Missing local RocksDB directory for " + tableBucket); + } + kvTablet = + kvManager.loadKvUnregistered( + directory, schemaGetter, this::onKvFlushComplete); + restoreStartOffset = released.getFlushedLogOffset(); + long cachedRows = released.getLifecycle().getCachedRowCount(); + rowCount = cachedRows == KvTablet.ROW_COUNT_DISABLED ? null : cachedRows; + autoIncIDRange = null; + } else if (optCompletedSnapshot.isPresent()) { LOG.info( "Use snapshot {} to restore kv tablet for {} of table {}.", optCompletedSnapshot.get(), @@ -1055,11 +991,11 @@ private InitKvResult initKvTablet() { downloadKvSnapshots(completedSnapshot, tabletDir.toPath()); // as we have downloaded kv files into the tablet dir, now, we can load it - createdTablet = + kvTablet = kvManager.loadKvUnregistered( tabletDir, schemaGetter, this::onKvFlushComplete); - checkNotNull(createdTablet, "kv tablet should not be null."); + checkNotNull(kvTablet, "kv tablet should not be null."); restoreStartOffset = completedSnapshot.getLogOffset(); rowCount = supportsExactRowCount(tableConfig) ? completedSnapshot.getRowCount() : null; @@ -1072,7 +1008,7 @@ private InitKvResult initKvTablet() { physicalPath); kvManager.createTabletDir(logTablet.getDataDir(), physicalPath, tableBucket); - createdTablet = + kvTablet = kvManager.createKvTabletUnregistered( physicalPath, tableBucket, @@ -1099,17 +1035,28 @@ private InitKvResult initKvTablet() { logTablet.updateMinRetainOffset(restoreStartOffset); if (isHistoricalPartition()) { - checkNotNull(createdTablet, "kv tablet should not be null.") - .advanceHistoricalCleanupOffset(restoreStartOffset); + checkNotNull(kvTablet, "kv tablet should not be null.") + .advanceHistoricalCleanupOffset( + released == null + ? restoreStartOffset + : Math.max(logTablet.getLakeLogEndOffset(), 0)); } - recoverKvTablet(createdTablet, restoreStartOffset, rowCount, autoIncIDRange); + recoverKvTablet(kvTablet, restoreStartOffset, rowCount, autoIncIDRange); } catch (Exception e) { - if (createdTablet != null) { - try { - createdTablet.close(); - } catch (Exception closeError) { - e.addSuppressed(closeError); + try { + if (kvTablet != null) { + kvTablet.close(); + } + if (released != null) { + FileUtils.deleteDirectory(released.getKvTabletDir()); } + } catch (Exception cleanupError) { + e.addSuppressed(cleanupError); + throw new KvStorageException("KV recovery cleanup failed for " + tableBucket, e); + } + if (released != null) { + LOG.warn("Local KV recovery failed for {}; restoring snapshot", tableBucket, e); + return initKvTablet(null); } throw new KvStorageException( String.format( @@ -1124,82 +1071,9 @@ private InitKvResult initKvTablet() { tableBucket, endTime - startTime); - return new InitKvResult(createdTablet, optCompletedSnapshot); - } - - private KvTablet reopenFromLocalData( - KvTablet sentinel, AtomicReference snapshotRef) { - checkNotNull(kvManager); - long startTime = clock.milliseconds(); - LOG.info( - "Reopening kv tablet from local data for {} of table {}.", - tableBucket, - physicalPath); - - long cachedFlushedLogOffset = sentinel.getCachedFlushedLogOffset(); - long cachedRowCount = sentinel.getCachedRowCount(); - - KvTablet createdTablet = null; - try { - File tabletDir = - kvManager.getTabletDirPath(logTablet.getDataDir(), physicalPath, tableBucket); - if (!new File(tabletDir, RocksDBKvBuilder.DB_INSTANCE_DIR_STRING).isDirectory()) { - throw new IOException("Local RocksDB directory is missing for " + tableBucket); - } - createdTablet = - kvManager.loadKvUnregistered(tabletDir, schemaGetter, this::onKvFlushComplete); - - if (isHistoricalPartition()) { - createdTablet.advanceHistoricalCleanupOffset( - logTablet.getLakeLogEndOffset() >= 0 - ? logTablet.getLakeLogEndOffset() - : 0L); - } - Long rowCount = cachedRowCount == KvTablet.ROW_COUNT_DISABLED ? null : cachedRowCount; - logTablet.updateMinRetainOffset(cachedFlushedLogOffset); - recoverKvTablet(createdTablet, cachedFlushedLogOffset, rowCount, null); - - long endTime = clock.milliseconds(); - LOG.info( - "Reopened kv tablet from local data for {} of table {} " - + "from log offset {}, cost {} ms.", - tableBucket, - physicalPath, - cachedFlushedLogOffset, - endTime - startTime); - - return createdTablet; - } catch (Exception e) { - LOG.warn( - "Failed to reopen from local data for {} of table {}, " - + "falling back to full init.", - tableBucket, - physicalPath, - e); - if (createdTablet != null) { - try { - createdTablet.close(); - } catch (Exception closeEx) { - e.addSuppressed(closeEx); - } - } - // Clean up the potentially corrupted local directory - File tabletDir = - kvManager.getTabletDirPath(logTablet.getDataDir(), physicalPath, tableBucket); - FileUtils.deleteDirectoryQuietly(tabletDir); - if (tabletDir.exists()) { - LOG.error( - "Failed to delete corrupted tablet dir {} during reopen fallback", - tabletDir.getAbsolutePath()); - } - // Fall back to full init (download snapshot) - InitKvResult result = initKvTablet(); - snapshotRef.set(result.snapshotUsed.orElse(null)); - return result.tablet; - } + return new InitKvResult(kvTablet, optCompletedSnapshot); } - /** Result of {@link #initKvTablet()}: the created tablet and the snapshot used (if any). */ private static class InitKvResult { final KvTablet tablet; final Optional snapshotUsed; @@ -1274,13 +1148,13 @@ private Optional getLatestSnapshot(TableBucket tableBucket) { } private void recoverKvTablet( - KvTablet tablet, + KvTablet kvTablet, long startRecoverLogOffset, @Nullable Long rowCount, @Nullable AutoIncIDRange autoIncIDRange) { TableConfig tableConfig = getTableConfig(); long start = clock.milliseconds(); - checkNotNull(tablet, "kv tablet should not be null."); + checkNotNull(kvTablet, "kv tablet should not be null."); try { KvRecoverHelper.KvRecoverContext recoverContext = new KvRecoverHelper.KvRecoverContext( @@ -1300,7 +1174,7 @@ private void recoverKvTablet( try { KvRecoverHelper kvRecoverHelper = new KvRecoverHelper( - tablet, + kvTablet, logTablet, startRecoverLogOffset, rowCount, @@ -1518,9 +1392,7 @@ public LogAppendInfo putRecordsToLeader( int requiredAcks) throws Exception { return withGuardedLeaderKv( - true, requiredAcks, - "KvTablet for leader access must not be null.", kv -> { if (!isLeader()) { throw new NotLeaderOrFollowerException( @@ -1566,9 +1438,7 @@ public LogAppendInfo putHistoricalRecordsToLeader( throws Exception { LocalValueLookupResult localLookupResult = withGuardedLeaderKv( - true, requiredAcks, - "Historical tablet must not be null.", kv -> { validateHistoricalWrite(expectedLeaderEpoch, requiredAcks); checkNotNull( @@ -1582,9 +1452,7 @@ public LogAppendInfo putHistoricalRecordsToLeader( localLookupResult.createValueLookup(lakeLookup); return withGuardedLeaderKv( - true, requiredAcks, - "Historical tablet must not be null.", kv -> { validateHistoricalWrite(expectedLeaderEpoch, requiredAcks); checkNotNull(kv, "KvTablet for the historical replica shouldn't be null."); @@ -1662,9 +1530,7 @@ private void validateHistoricalWrite(int expectedLeaderEpoch, int requiredAcks) public List lookupHistoricalLocal( String originalPartitionName, List keys) throws Exception { return withGuardedLeaderKv( - false, 0, - "KvTablet for leader access must not be null.", kv -> { if (!isLeader()) { throw new NotLeaderOrFollowerException( @@ -1904,9 +1770,7 @@ public List lookups(List keys) { "the primary key table not exists for " + tableBucket); } return withGuardedLeaderKv( - false, 0, - "KvTablet for leader access must not be null.", kv -> { try { if (!isLeader()) { @@ -1941,9 +1805,7 @@ public List lookupsFromBufferOrKv(List keys) { "the primary key table not exists for " + tableBucket); } return withGuardedLeaderKv( - false, 0, - "KvTablet for leader access must not be null.", kv -> { try { if (!isLeader()) { @@ -1973,9 +1835,7 @@ public List prefixLookup(byte[] prefixKey) { } return withGuardedLeaderKv( - false, 0, - "KvTablet for leader access must not be null.", kv -> { try { if (!isLeader()) { @@ -2005,9 +1865,7 @@ public DefaultValueRecordBatch limitKvScan(int limit) { } return withGuardedLeaderKv( - false, 0, - "KvTablet for leader access must not be null.", kv -> { try { if (!isLeader()) { @@ -2056,9 +1914,7 @@ public OpenScanResult openScan( } return withGuardedLeaderKv( - false, 0, - "KvTablet for leader access must not be null.", kv -> { if (!isLeader()) { throw new NotLeaderOrFollowerException( @@ -2084,24 +1940,18 @@ public OpenScanResult openScan( } private T withGuardedLeaderKv( - boolean validateInSyncReplicas, - int requiredAcks, - String nullKvMessage, - GuardedKvOperation action) - throws E { + int requiredAcks, GuardedKvOperation action) throws E { while (true) { KvTablet kv = inReadLock( leaderIsrUpdateLock, () -> { ensureLeaderForKvAccess(); - if (validateInSyncReplicas) { - validateInSyncReplicaSize(requiredAcks); - } - return checkNotNull(kvTablet, nullKvMessage); + validateInSyncReplicaSize(requiredAcks); + return checkNotNull(kvTablet, "Leader KV tablet is missing."); }); boolean openingKv = kv.isLazyMode() && !kv.isLazyOpen(); - try (KvTablet.Guard ignored = kv.acquireGuard()) { + try (KvTablet.Guard guard = kv.acquireGuard()) { // A leadership transition drains pins while holding the write lock. Never wait // for that lock with a pin: release it and retry after the transition instead. if (leaderIsrUpdateLock.readLock().tryLock()) { @@ -2111,10 +1961,8 @@ private T withGuardedLeaderKv( throw new NotLeaderOrFollowerException( "KV tablet changed while opening " + tableBucket); } - if (validateInSyncReplicas) { - validateInSyncReplicaSize(requiredAcks); - } - return action.apply(kv); + validateInSyncReplicaSize(requiredAcks); + return action.apply(guard.getTablet()); } finally { leaderIsrUpdateLock.readLock().unlock(); } @@ -2215,15 +2063,20 @@ public Tuple2 checkEnoughReplicasReachOffset(long requiredOffse } public long getRowCount() { - KvTablet kv = inReadLock(leaderIsrUpdateLock, () -> kvTablet); - - if (kv != null) { - return kv.getRowCount(); - } else if (isKvTable()) { - return 0L; - } else { - return logTablet.getRowCount(); - } + return inReadLock( + leaderIsrUpdateLock, + () -> { + KvTablet kv = this.kvTablet; + if (kv != null) { + // return materialized row count for primary key table + return kv.getRowCount(); + } else if (isKvTable()) { + return 0L; + } else { + // return log row count for non-primary key table + return logTablet.getRowCount(); + } + }); } private static boolean supportsExactRowCount(TableConfig tableConfig) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index 3ac4715f3bd..c11d2fe418f 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -88,10 +88,8 @@ import org.apache.fluss.server.entity.StopReplicaData; import org.apache.fluss.server.entity.StopReplicaResultForBucket; import org.apache.fluss.server.entity.UserContext; -import org.apache.fluss.server.kv.KvIdleReleaseController; import org.apache.fluss.server.kv.KvManager; import org.apache.fluss.server.kv.KvSnapshotResource; -import org.apache.fluss.server.kv.KvTablet; import org.apache.fluss.server.kv.scan.ScannerManager; import org.apache.fluss.server.kv.snapshot.CompletedKvSnapshotCommitter; import org.apache.fluss.server.kv.snapshot.DefaultSnapshotContext; @@ -154,7 +152,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -254,8 +251,7 @@ public class ReplicaManager implements ServerReconfigurable { private final ScannerManager scannerManager; private final HistoricalPartitionManager historicalPartitionManager; - private final boolean kvIdleReleaseEnabled; - private @Nullable KvIdleReleaseController kvIdleReleaseController; + private final long kvIdleTimeoutMs; private @Nullable ScheduledExecutorService kvIdleReleaseScheduler; public ReplicaManager( @@ -391,29 +387,12 @@ public ReplicaManager( dataDirVolumeBytes, scheduler); - this.kvIdleReleaseEnabled = kvManager != null && kvManager.isLazyOpenEnabled(); - if (kvIdleReleaseEnabled) { - long closeIdleIntervalMs = conf.get(ConfigOptions.KV_LAZY_OPEN_IDLE_TIMEOUT).toMillis(); - checkArgument( - closeIdleIntervalMs > 0, - "kv.lazy-open.idle-timeout must be positive, got: %s ms", - closeIdleIntervalMs); - this.kvIdleReleaseScheduler = + this.kvIdleTimeoutMs = conf.get(ConfigOptions.KV_LAZY_OPEN_IDLE_TIMEOUT).toMillis(); + if (kvManager != null && kvManager.isLazyOpenEnabled()) { + checkArgument(kvIdleTimeoutMs > 0, "kv.lazy-open.idle-timeout must be positive"); + kvIdleReleaseScheduler = Executors.newSingleThreadScheduledExecutor( new ExecutorThreadFactory("kv-idle-release")); - this.kvIdleReleaseController = - new KvIdleReleaseController( - kvIdleReleaseScheduler, - () -> - onlineReplicas() - .filter(Replica::isKvTable) - .map(Replica::getKvTablet) - .filter(Objects::nonNull) - .filter(KvTablet::isLazyOpen) - .collect(Collectors.toList()), - clock, - IDLE_RELEASE_CHECK_INTERVAL_MS, - closeIdleIntervalMs); } registerMetrics(); } @@ -432,9 +411,12 @@ public void startup() { // Start periodic disk usage monitoring (initial + periodic sampling) localDiskManager.startDiskUsageMonitor(scheduler); - // Start idle release controller if enabled - if (kvIdleReleaseController != null) { - kvIdleReleaseController.start(); + if (kvIdleReleaseScheduler != null) { + kvIdleReleaseScheduler.scheduleWithFixedDelay( + () -> kvManager.releaseIdleTablets(kvIdleTimeoutMs, clock.milliseconds()), + IDLE_RELEASE_CHECK_INTERVAL_MS, + IDLE_RELEASE_CHECK_INTERVAL_MS, + TimeUnit.MILLISECONDS); } } @@ -2718,10 +2700,6 @@ public Replica getReplica() { public static final class OfflineReplica implements HostedReplica {} public void shutdown() throws InterruptedException { - // Shut down the idle release controller if it was created - if (kvIdleReleaseController != null) { - kvIdleReleaseController.close(); - } if (kvIdleReleaseScheduler != null) { ExecutorUtils.gracefulShutdown(5, TimeUnit.SECONDS, kvIdleReleaseScheduler); } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvIdleReleaseControllerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvIdleReleaseControllerTest.java deleted file mode 100644 index 0bfda40c03a..00000000000 --- a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvIdleReleaseControllerTest.java +++ /dev/null @@ -1,332 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.server.kv; - -import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.config.Configuration; -import org.apache.fluss.config.TableConfig; -import org.apache.fluss.metadata.KvFormat; -import org.apache.fluss.metadata.LogFormat; -import org.apache.fluss.metadata.PhysicalTablePath; -import org.apache.fluss.metadata.SchemaInfo; -import org.apache.fluss.metadata.TableBucket; -import org.apache.fluss.metadata.TablePath; -import org.apache.fluss.record.KvRecord; -import org.apache.fluss.record.KvRecordBatch; -import org.apache.fluss.record.KvRecordTestUtils; -import org.apache.fluss.record.TestData; -import org.apache.fluss.record.TestingSchemaGetter; -import org.apache.fluss.server.log.LogManager; -import org.apache.fluss.server.log.LogTablet; -import org.apache.fluss.server.metrics.group.TestingMetricGroups; -import org.apache.fluss.server.storage.LocalDiskManager; -import org.apache.fluss.server.zk.NOPErrorHandler; -import org.apache.fluss.server.zk.ZooKeeperClient; -import org.apache.fluss.server.zk.ZooKeeperExtension; -import org.apache.fluss.testutils.common.AllCallbackWrapper; -import org.apache.fluss.testutils.common.CommonTestUtils; -import org.apache.fluss.testutils.common.ManuallyTriggeredScheduledExecutorService; -import org.apache.fluss.types.RowType; -import org.apache.fluss.utils.clock.ManualClock; -import org.apache.fluss.utils.clock.SystemClock; -import org.apache.fluss.utils.concurrent.FlussScheduler; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.api.io.TempDir; - -import java.io.File; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static org.apache.fluss.compression.ArrowCompressionInfo.DEFAULT_COMPRESSION; -import static org.apache.fluss.record.TestData.DATA1_SCHEMA_PK; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatCode; - -/** Test for {@link KvIdleReleaseController}. */ -final class KvIdleReleaseControllerTest { - - @RegisterExtension - public static final AllCallbackWrapper ZOO_KEEPER_EXTENSION_WRAPPER = - new AllCallbackWrapper<>(new ZooKeeperExtension()); - - private static final short SCHEMA_ID = 1; - private static final RowType BASE_ROW_TYPE = TestData.DATA1_ROW_TYPE; - private static final long IDLE_INTERVAL_MS = 60_000; - private static final long CHECK_INTERVAL_MS = 10_000; - - private final KvRecordTestUtils.KvRecordBatchFactory batchFactory = - KvRecordTestUtils.KvRecordBatchFactory.of(SCHEMA_ID); - private final KvRecordTestUtils.KvRecordFactory recordFactory = - KvRecordTestUtils.KvRecordFactory.of(BASE_ROW_TYPE); - - private static ZooKeeperClient zkClient; - - private @TempDir File tempDir; - - private LocalDiskManager localDiskManager; - private LogManager logManager; - private KvManager kvManager; - private Configuration conf; - private ManualClock clock; - private ManuallyTriggeredScheduledExecutorService scheduler; - private List allTablets; - private int nextTableId = 30001; - - @BeforeAll - static void baseBeforeAll() { - zkClient = - ZOO_KEEPER_EXTENSION_WRAPPER - .getCustomExtension() - .getZooKeeperClient(NOPErrorHandler.INSTANCE); - } - - @BeforeEach - void setup() throws Exception { - conf = new Configuration(); - conf.setString(ConfigOptions.DATA_DIR, tempDir.getAbsolutePath()); - conf.set(ConfigOptions.KV_LAZY_OPEN_ENABLED, true); - - conf.set(ConfigOptions.TABLET_SERVER_ID, 1); - localDiskManager = LocalDiskManager.create(conf); - logManager = - LogManager.create( - conf, - zkClient, - new FlussScheduler(1), - SystemClock.getInstance(), - TestingMetricGroups.TABLET_SERVER_METRICS, - localDiskManager); - kvManager = - KvManager.create( - conf, - zkClient, - logManager, - TestingMetricGroups.TABLET_SERVER_METRICS, - localDiskManager); - kvManager.startup(); - - clock = new ManualClock(System.currentTimeMillis()); - scheduler = new ManuallyTriggeredScheduledExecutorService(); - allTablets = new ArrayList<>(); - } - - @AfterEach - void tearDown() throws Exception { - if (kvManager != null) { - kvManager.shutdown(); - } - if (logManager != null) { - logManager.shutdown(); - } - if (localDiskManager != null) { - localDiskManager.close(); - } - } - - // ---- Helper methods ---- - - private KvTablet createAndOpenTablet(int tableId, int bucketId) throws Exception { - TablePath tablePath = TablePath.of("db1", "t_idle_" + tableId); - PhysicalTablePath physicalPath = - PhysicalTablePath.of(tablePath.getDatabaseName(), tablePath.getTableName(), null); - TableBucket tableBucket = new TableBucket((long) tableId, bucketId); - LogTablet logTablet = - logManager.getOrCreateLog( - tempDir, physicalPath, tableBucket, LogFormat.ARROW, 1, true); - - KvTablet sentinel = - KvTablet.createLazySentinel( - physicalPath, - tableBucket, - logTablet, - TestingMetricGroups.TABLET_SERVER_METRICS); - - KvTabletLazyLifecycle lifecycle = sentinel.getLifecycle(); - lifecycle.configureLazyOpen( - clock, - kvManager.getOpenSemaphore(), - kvManager.getOpenTimeoutMs(), - kvManager.getFailedBackoffBaseMs(), - kvManager.getFailedBackoffMaxMs(), - KvManager.RELEASE_DRAIN_TIMEOUT_MS); - lifecycle.setLeaderEpochSupplier(() -> 1); - lifecycle.setBucketEpochSupplier(() -> 1); - lifecycle.setTabletDirSupplier( - () -> - kvManager.getTabletDirPath( - logTablet.getDataDir(), physicalPath, tableBucket)); - lifecycle.setOpenCallback( - hasLocal -> - kvManager.createKvTabletUnregistered( - physicalPath, - tableBucket, - logTablet, - KvFormat.COMPACTED, - new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), - new TableConfig(new Configuration()), - DEFAULT_COMPRESSION, - null)); - lifecycle.setCommitCallback(kv -> {}); - lifecycle.setReleaseCallback(kv -> {}); - lifecycle.setDropCallback(kv -> {}); - - // Open the tablet via acquireGuard - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - // Write and flush so releaseKv() won't be rejected - KvRecord record = - recordFactory.ofRecord( - ("k_" + tableId + "_" + bucketId).getBytes(), - new Object[] {tableId, "v"}); - KvRecordBatch batch = batchFactory.ofRecords(Collections.singletonList(record)); - sentinel.putAsLeader(batch, null); - long offset = logTablet.localLogEndOffset(); - sentinel.requestFlush(offset, NOPErrorHandler.INSTANCE); - CommonTestUtils.waitUntil( - () -> sentinel.getFlushedLogOffset() >= offset, - Duration.ofSeconds(10), - "KV operation did not complete"); - } - - kvManager.registerKv(tableBucket, sentinel); - return sentinel; - } - - private Supplier> openTabletSupplier() { - return () -> allTablets.stream().filter(KvTablet::isLazyOpen).collect(Collectors.toList()); - } - - private KvIdleReleaseController createController() { - return new KvIdleReleaseController( - scheduler, openTabletSupplier(), clock, CHECK_INTERVAL_MS, IDLE_INTERVAL_MS); - } - - // ---- Tests ---- - - @Test - void testNoReleaseWhenAllActive() throws Exception { - KvTablet tablet1 = createAndOpenTablet(nextTableId++, 0); - KvTablet tablet2 = createAndOpenTablet(nextTableId++, 0); - allTablets.add(tablet1); - allTablets.add(tablet2); - - // Don't advance clock — tablets are recently accessed - KvIdleReleaseController controller = createController(); - controller.checkAndRelease(); - - assertThat(tablet1.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - assertThat(tablet2.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - } - - @Test - void testReleaseAllIdleTablets() throws Exception { - KvTablet tablet1 = createAndOpenTablet(nextTableId++, 0); - KvTablet tablet2 = createAndOpenTablet(nextTableId++, 0); - allTablets.add(tablet1); - allTablets.add(tablet2); - - // Advance clock past idle interval - clock.advanceTime(IDLE_INTERVAL_MS + 1, TimeUnit.MILLISECONDS); - - KvIdleReleaseController controller = createController(); - controller.checkAndRelease(); - - assertThat(tablet1.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); - assertThat(tablet2.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); - } - - @Test - void testOnlyIdleTabletsReleased() throws Exception { - // Create tablet1 first, then advance clock, then create tablet2 - KvTablet tablet1 = createAndOpenTablet(nextTableId++, 0); - allTablets.add(tablet1); - - // Advance clock so tablet1 becomes idle but tablet2 (created after) won't be - clock.advanceTime(IDLE_INTERVAL_MS + 1, TimeUnit.MILLISECONDS); - - KvTablet tablet2 = createAndOpenTablet(nextTableId++, 0); - allTablets.add(tablet2); - - KvIdleReleaseController controller = createController(); - controller.checkAndRelease(); - - // tablet1 is idle — released - assertThat(tablet1.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); - // tablet2 is recent — stays open - assertThat(tablet2.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - } - - @Test - void testSkipTabletWithActivePins() throws Exception { - KvTablet tablet1 = createAndOpenTablet(nextTableId++, 0); - KvTablet tablet2 = createAndOpenTablet(nextTableId++, 0); - allTablets.add(tablet1); - allTablets.add(tablet2); - - // Hold a guard on tablet1 (don't close it) — this pins it - try (KvTablet.Guard heldGuard = tablet1.acquireGuard()) { - // Advance clock past idle - clock.advanceTime(IDLE_INTERVAL_MS + 1, TimeUnit.MILLISECONDS); - - KvIdleReleaseController controller = createController(); - controller.checkAndRelease(); - - // tablet1 is pinned — should NOT be released - assertThat(tablet1.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - // tablet2 is idle and unpinned — should be released - assertThat(tablet2.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); - } - } - - @Test - void testContinuesAfterReleaseFailure() throws Exception { - KvTablet tablet1 = createAndOpenTablet(nextTableId++, 0); - KvTablet tablet2 = createAndOpenTablet(nextTableId++, 0); - allTablets.add(tablet1); - allTablets.add(tablet2); - - // Make tablet1's releaseCallback throw so releaseKv() returns false - tablet1.getLifecycle() - .setReleaseCallback( - kv -> { - throw new RuntimeException("simulated release failure"); - }); - - // Advance clock past idle - clock.advanceTime(IDLE_INTERVAL_MS + 1, TimeUnit.MILLISECONDS); - - KvIdleReleaseController controller = createController(); - assertThatCode(() -> controller.checkAndRelease()).doesNotThrowAnyException(); - - // tablet1 release failed — should rollback to OPEN - assertThat(tablet1.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - // tablet2 should still be released despite tablet1's failure - assertThat(tablet2.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); - tablet1.getLifecycle().setReleaseCallback(kv -> {}); - } -} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvManagerTest.java index 1df2d336a42..6491739455b 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvManagerTest.java @@ -22,6 +22,7 @@ import org.apache.fluss.config.MemorySize; import org.apache.fluss.config.TableConfig; import org.apache.fluss.exception.FlussRuntimeException; +import org.apache.fluss.exception.KvStorageException; import org.apache.fluss.metadata.KvFormat; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.PhysicalTablePath; @@ -50,16 +51,20 @@ import org.apache.fluss.server.zk.ZooKeeperExtension; import org.apache.fluss.server.zk.data.TableRegistration; import org.apache.fluss.testutils.common.AllCallbackWrapper; +import org.apache.fluss.testutils.common.CommonTestUtils; import org.apache.fluss.types.RowType; import org.apache.fluss.utils.ByteArraySlice; import org.apache.fluss.utils.FlussPaths; +import org.apache.fluss.utils.clock.ManualClock; import org.apache.fluss.utils.clock.SystemClock; import org.apache.fluss.utils.concurrent.FlussScheduler; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -80,17 +85,25 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.IntSupplier; import static org.apache.fluss.compression.ArrowCompressionInfo.DEFAULT_COMPRESSION; import static org.apache.fluss.record.TestData.DATA1_SCHEMA_PK; import static org.apache.fluss.record.TestData.DATA2_SCHEMA; import static org.apache.fluss.server.kv.KvTabletTestUtils.flushAndWait; +import static org.apache.fluss.testutils.DataTestUtils.compactedRow; import static org.apache.fluss.testutils.common.CommonTestUtils.waitUntil; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -133,10 +146,13 @@ static void baseBeforeAll() { } @BeforeEach - void setup() throws Exception { + void setup(TestInfo testInfo) throws Exception { conf = new Configuration(); conf.setString(ConfigOptions.DATA_DIR, tempDir.getAbsolutePath()); conf.set(ConfigOptions.TABLET_SERVER_ID, 1); + conf.set( + ConfigOptions.KV_LAZY_OPEN_ENABLED, + testInfo.getTestClass().orElse(KvManagerTest.class) == LazyLifecycleTest.class); String dbName = "db1"; tablePath1 = TablePath.of(dbName, "t1"); @@ -783,129 +799,6 @@ void testDropKv(String partitionName) throws Exception { assertThat(kvManager.getKv(tableBucket1)).isPresent(); } - @ParameterizedTest - @MethodSource("partitionProvider") - void testRegisterKv(String partitionName) throws Exception { - initTableBuckets(partitionName); - PhysicalTablePath physicalTablePath = - PhysicalTablePath.of( - tablePath1.getDatabaseName(), tablePath1.getTableName(), partitionName); - LogTablet logTablet = - logManager.getOrCreateLog( - tempDir, physicalTablePath, tableBucket1, LogFormat.ARROW, 1, true); - - // Create a lazy sentinel and register it - KvTablet sentinel = - KvTablet.createLazySentinel(physicalTablePath, tableBucket1, logTablet, null); - kvManager.registerKv(tableBucket1, sentinel); - - // verify registered - assertThat(kvManager.getKv(tableBucket1)).isPresent(); - assertThat(kvManager.getKv(tableBucket1).get()).isSameAs(sentinel); - } - - @Test - void testRegisterKvDuplicate() throws Exception { - initTableBuckets(null); - PhysicalTablePath physicalTablePath = - PhysicalTablePath.of(tablePath1.getDatabaseName(), tablePath1.getTableName(), null); - LogTablet logTablet = - logManager.getOrCreateLog( - tempDir, physicalTablePath, tableBucket1, LogFormat.ARROW, 1, true); - - KvTablet sentinel = - KvTablet.createLazySentinel(physicalTablePath, tableBucket1, logTablet, null); - kvManager.registerKv(tableBucket1, sentinel); - - // registering again should throw - KvTablet sentinel2 = - KvTablet.createLazySentinel(physicalTablePath, tableBucket1, logTablet, null); - org.assertj.core.api.Assertions.assertThatThrownBy( - () -> kvManager.registerKv(tableBucket1, sentinel2)) - .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("already registered"); - } - - @ParameterizedTest - @MethodSource("partitionProvider") - void testUnregisterKv(String partitionName) throws Exception { - initTableBuckets(partitionName); - KvTablet tablet = getOrCreateKv(tablePath1, partitionName, tableBucket1); - assertThat(kvManager.getKv(tableBucket1)).isPresent(); - - kvManager.unregisterKv(tableBucket1, tablet); - assertThat(kvManager.getKv(tableBucket1)).isNotPresent(); - tablet.close(); - } - - @ParameterizedTest - @MethodSource("partitionProvider") - void testCreateKvTabletUnregistered(String partitionName) throws Exception { - initTableBuckets(partitionName); - PhysicalTablePath physicalTablePath = - PhysicalTablePath.of( - tablePath1.getDatabaseName(), tablePath1.getTableName(), partitionName); - LogTablet logTablet = - logManager.getOrCreateLog( - tempDir, physicalTablePath, tableBucket1, LogFormat.ARROW, 1, true); - - KvTablet tablet = - kvManager.createKvTabletUnregistered( - physicalTablePath, - tableBucket1, - logTablet, - KvFormat.COMPACTED, - new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, 1)), - new TableConfig(new Configuration()), - DEFAULT_COMPRESSION, - null); - - // tablet should be created but NOT registered - assertThat(tablet.getKvTabletDir()).exists(); - assertThat(kvManager.getKv(tableBucket1)).isNotPresent(); - tablet.close(); - } - - @ParameterizedTest - @MethodSource("partitionProvider") - void testGetTabletDirPath(String partitionName) throws Exception { - initTableBuckets(partitionName); - PhysicalTablePath physicalTablePath = - PhysicalTablePath.of( - tablePath1.getDatabaseName(), tablePath1.getTableName(), partitionName); - - File dirPath = kvManager.getTabletDirPath(tempDir, physicalTablePath, tableBucket1); - // directory should not be created by this call - assertThat(dirPath).doesNotExist(); - - // Assert the exact expected path structure - File expectedDir; - if (partitionName == null) { - // Non-partitioned: {dataDir}/{db}/{table}-{tableId}/kv-{bucket} - expectedDir = - new File( - tempDir, - "db1/t1-" - + tableBucket1.getTableId() - + "/kv-" - + tableBucket1.getBucket()); - } else { - // Partitioned: {dataDir}/{db}/{table}-{tableId}/{partition}-p{partitionId}/kv-{bucket} - expectedDir = - new File( - tempDir, - "db1/t1-" - + tableBucket1.getTableId() - + "/" - + partitionName - + "-p" - + tableBucket1.getPartitionId() - + "/kv-" - + tableBucket1.getBucket()); - } - assertThat(dirPath).isEqualTo(expectedDir); - } - @Test void testKvRunConcurrentlyForDifferentBuckets() throws Exception { initTableBuckets(null); @@ -1251,4 +1144,543 @@ private void unblock() { unblock.countDown(); } } + + private static FutureTask start(Callable action) { + FutureTask task = new FutureTask<>(action); + new Thread(task, "lazy-lifecycle-test").start(); + return task; + } + + private static class BlockingCallback implements AutoCloseable { + private final CountDownLatch entered = new CountDownLatch(1); + private final CountDownLatch finish = new CountDownLatch(1); + + void run() { + entered.countDown(); + try { + finish.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + void awaitEntered() throws Exception { + assertThat(entered.await(5, TimeUnit.SECONDS)).isTrue(); + } + + @Override + public void close() { + finish.countDown(); + } + } + + @Nested + class LazyLifecycleTest { + private PhysicalTablePath physicalPath; + private TableBucket tableBucket; + private LogTablet logTablet; + private ManualClock clock; + private KvTablet sentinel; + private KvTabletLazyLifecycle lifecycle; + private int baseLazy; + private int baseOpen; + private int baseFailed; + private KvTabletLazyLifecycle.OpenCallback openCallback; + private Consumer commitCallback = kv -> {}; + private Consumer releaseCallback = kv -> {}; + private IntSupplier leaderEpochSupplier = () -> 1; + private IntSupplier bucketEpochSupplier = () -> 1; + + @BeforeEach + void setupLifecycle() throws Exception { + assertThat(kvManager.isLazyOpenEnabled()).isTrue(); + TablePath tablePath = TablePath.of("db1", "t_lazy"); + physicalPath = + PhysicalTablePath.of( + tablePath.getDatabaseName(), tablePath.getTableName(), null); + tableBucket = new TableBucket(20001L, 0); + logTablet = + logManager.getOrCreateLog( + tempDir, physicalPath, tableBucket, LogFormat.ARROW, 1, true); + TabletServerMetricGroup metrics = TestingMetricGroups.TABLET_SERVER_METRICS; + baseLazy = metrics.kvTabletLazyCount().get(); + baseOpen = metrics.kvTabletOpenCount().get(); + baseFailed = metrics.kvTabletFailedCount().get(); + clock = new ManualClock(System.currentTimeMillis()); + openCallback = local -> openTablet(sentinel); + sentinel = createSentinel(); + lifecycle = configureLifecycle(sentinel); + } + + private KvTablet createSentinel() { + KvTablet tablet = kvManager.createLazyTablet(physicalPath, tableBucket, logTablet); + kvManager.registerKv(tableBucket, tablet); + return tablet; + } + + private KvTabletLazyLifecycle configureLifecycle(KvTablet sentinel) { + KvTabletLazyLifecycle lifecycle = sentinel.getLifecycle(); + lifecycle.configureTiming(clock, new java.util.concurrent.Semaphore(10), 100); + lifecycle.configure( + () -> leaderEpochSupplier.getAsInt(), + () -> bucketEpochSupplier.getAsInt(), + local -> openCallback.doOpen(local), + kv -> commitCallback.accept(kv), + kv -> releaseCallback.accept(kv)); + return lifecycle; + } + + private KvTablet openTablet(KvTablet owner) throws Exception { + return kvManager.createKvTabletUnregistered( + physicalPath, + owner.getTableBucket(), + owner.logTablet, + KvFormat.COMPACTED, + new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, schemaId)), + new TableConfig(new Configuration()), + DEFAULT_COMPRESSION, + null); + } + + private KvTablet secondTablet() throws Exception { + TableBucket bucket = new TableBucket(tableBucket.getTableId(), 1); + LogTablet log = + logManager.getOrCreateLog( + tempDir, physicalPath, bucket, LogFormat.ARROW, 1, true); + KvTablet second = kvManager.createLazyTablet(physicalPath, bucket, log); + second.getLifecycle() + .configureTiming(clock, new java.util.concurrent.Semaphore(10), 100); + second.getLifecycle() + .configure(() -> 1, () -> 1, local -> openTablet(second), kv -> {}, kv -> {}); + kvManager.registerKv(bucket, second); + return second; + } + + @ParameterizedTest + @ValueSource(strings = {"active", "idle", "recent", "pinned", "failed"}) + void testIdleRelease(String condition) throws Exception { + KvTablet second = secondTablet(); + assertGauges(2, 0, 0); + try (KvTablet.Guard ignored = sentinel.acquireGuard(); + KvTablet.Guard other = second.acquireGuard()) { + writeRecord(); + flushAndAwait(sentinel); + } + assertGauges(0, 2, 0); + KvTablet.Guard pin = condition.equals("pinned") ? sentinel.acquireGuard() : null; + if (!condition.equals("active")) { + clock.advanceTime(61, TimeUnit.SECONDS); + } + if (condition.equals("recent")) { + try (KvTablet.Guard ignored = sentinel.acquireGuard()) { + assertThat(lifecycle.getLastAccessTimestamp()).isEqualTo(clock.milliseconds()); + } + } + if (condition.equals("failed")) { + releaseCallback = + kv -> { + throw new IllegalStateException("release failed"); + }; + } + try { + kvManager.releaseIdleTablets(60_000, clock.milliseconds()); + assertThat(sentinel.isLazyOpen()).isEqualTo(!condition.equals("idle")); + assertThat(second.isLazyOpen()).isEqualTo(condition.equals("active")); + int open = (sentinel.isLazyOpen() ? 1 : 0) + (second.isLazyOpen() ? 1 : 0); + assertGauges(2 - open, open, 0); + if (condition.equals("failed")) { + assertValue(); + releaseCallback = kv -> {}; + assertThat(sentinel.releaseKv()).isTrue(); + assertGauges(2, 0, 0); + assertValue(); + assertGauges(1, 1, 0); + } + } finally { + if (pin != null) { + pin.close(); + } + releaseCallback = kv -> {}; + } + } + + private void open() { + try (KvTablet.Guard ignored = sentinel.acquireGuard()) { + assertThat(sentinel.getRocksDBKv()).isNotNull(); + } + } + + private void awaitState(KvTabletLazyLifecycle.LazyState state) throws Exception { + CommonTestUtils.waitUntil( + () -> lifecycle.getLazyState() == state, + Duration.ofSeconds(5), + "Waiting for " + state); + } + + private void assertGauges(int lazy, int open, int failed) { + TabletServerMetricGroup metrics = TestingMetricGroups.TABLET_SERVER_METRICS; + assertThat(metrics.kvTabletLazyCount().get() - baseLazy).isEqualTo(lazy); + assertThat(metrics.kvTabletOpenCount().get() - baseOpen).isEqualTo(open); + assertThat(metrics.kvTabletFailedCount().get() - baseFailed).isEqualTo(failed); + } + + private void assertValue() throws Exception { + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + assertThat(guard.getTablet().multiGet(Collections.singletonList("k1".getBytes()))) + .extracting(ByteArraySlice::toByteArray) + .containsExactly( + ValueEncoder.encodeValue( + schemaId, + compactedRow(baseRowType, new Object[] {1, "a"}))); + } + } + + private void writeRecord() throws Exception { + sentinel.requireOpenedTablet() + .putAsLeader( + kvRecordBatchFactory.ofRecords( + Collections.singletonList( + kvRecordFactory.ofRecord( + "k1".getBytes(), new Object[] {1, "a"}))), + null); + } + + private void flushAndAwait(KvTablet tablet) throws Exception { + long offset = logTablet.localLogEndOffset(); + tablet.requestFlush(offset, NOPErrorHandler.INSTANCE); + CommonTestUtils.waitUntil( + () -> tablet.getFlushedLogOffset() >= offset, + Duration.ofSeconds(10), + "KV operation did not complete"); + } + + @ParameterizedTest + @CsvSource({"release,false", "release,true", "failure,true", "commit,true"}) + void testCloseWaitsForCallbacks(String phase, boolean delete) throws Exception { + boolean commit = phase.equals("commit"); + try (BlockingCallback blocked = new BlockingCallback()) { + if (commit) { + commitCallback = kv -> blocked.run(); + } else { + open(); + AtomicInteger cleanupCalls = new AtomicInteger(); + releaseCallback = + kv -> { + blocked.run(); + if (cleanupCalls.incrementAndGet() == 1 + && phase.equals("failure")) { + throw new IllegalStateException("release failed"); + } + }; + } + FutureTask work = + start( + () -> { + if (commit) { + assertThatThrownBy(sentinel::acquireGuard) + .isInstanceOf(KvStorageException.class); + return null; + } + return sentinel.releaseKv(); + }); + blocked.awaitEntered(); + FutureTask close = + start( + () -> { + if (delete) { + sentinel.drop(); + } else { + sentinel.close(); + } + return null; + }); + awaitState(KvTabletLazyLifecycle.LazyState.CLOSING); + clock.advanceTime(1, TimeUnit.SECONDS); + assertThatThrownBy(() -> close.get(200, TimeUnit.MILLISECONDS)) + .isInstanceOf(TimeoutException.class); + blocked.close(); + work.get(5, TimeUnit.SECONDS); + close.get(5, TimeUnit.SECONDS); + } + releaseCallback = kv -> {}; + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); + assertThat(sentinel.getRocksDBKv()).isNull(); + assertThat(sentinel.getKvTabletDir().exists()).isEqualTo(!delete); + assertThatThrownBy(sentinel::acquireGuard).isInstanceOf(KvStorageException.class); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testCloseWaitsForFencedOpenCleanup(boolean shutdown) throws Exception { + try (BlockingCallback blocked = new BlockingCallback()) { + AtomicReference candidate = new AtomicReference<>(); + AtomicReference lease = new AtomicReference<>(); + openCallback = + hasLocal -> { + KvTablet opened = openTablet(sentinel); + candidate.set(opened); + lease.set(opened.getRocksDBKv().getResourceGuard().acquireResource()); + blocked.run(); + return opened; + }; + FutureTask open = + start( + () -> { + assertThatThrownBy(sentinel::acquireGuard) + .isInstanceOf(RuntimeException.class); + return null; + }); + blocked.awaitEntered(); + FutureTask close = + start( + () -> { + if (shutdown) { + kvManager.shutdown(); + } else { + kvManager.dropKv(tableBucket); + } + return null; + }); + try { + awaitState(KvTabletLazyLifecycle.LazyState.CLOSING); + blocked.close(); + CommonTestUtils.waitUntil( + () -> candidate.get().getRocksDBKv().getResourceGuard().isClosed(), + Duration.ofSeconds(5), + "Candidate cleanup did not start"); + assertThatThrownBy(() -> close.get(200, TimeUnit.MILLISECONDS)) + .isInstanceOf(TimeoutException.class); + assertThat(kvManager.getKv(tableBucket)).contains(sentinel); + assertThatThrownBy(() -> kvManager.registerKv(tableBucket, sentinel)) + .isInstanceOf(IllegalStateException.class); + } finally { + blocked.close(); + if (lease.get() != null) { + lease.get().close(); + } + open.get(5, TimeUnit.SECONDS); + close.get(5, TimeUnit.SECONDS); + } + assertThat(lifecycle.getLazyState()) + .isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); + assertThat(sentinel.getRocksDBKv()).isNull(); + assertThat(sentinel.getKvTabletDir()).doesNotExist(); + if (shutdown) { + kvManager = null; + } else { + assertThat(kvManager.getKv(tableBucket)).isEmpty(); + KvTablet replacement = createSentinel(); + openCallback = local -> openTablet(replacement); + configureLifecycle(replacement); + try (KvTablet.Guard ignored = replacement.acquireGuard()) { + assertThat(replacement.getKvTabletDir()).isDirectory(); + } + } + } + } + + @Test + void testDropAfterCloseDeletesLocalDataOnlyOnce() throws Exception { + try (KvTablet.Guard ignored = sentinel.acquireGuard()) { + assertThat(sentinel.getKvTabletDir()).isDirectory(); + } + sentinel.close(); + assertThat(sentinel.getKvTabletDir()).isDirectory(); + sentinel.drop(); + assertThat(sentinel.getKvTabletDir()).doesNotExist(); + kvManager.dropKv(tableBucket); + KvTablet replacement = createSentinel(); + configureLifecycle(replacement); + try (KvTablet.Guard ignored = replacement.acquireGuard()) { + sentinel.drop(); + assertThat(replacement.getKvTabletDir()).isDirectory(); + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testReleaseChecksWritesCompletedDuringDrain(boolean flush) throws Exception { + FutureTask release; + try (KvTablet.Guard ignored = sentinel.acquireGuard()) { + release = start(sentinel::releaseKv); + awaitState(KvTabletLazyLifecycle.LazyState.RELEASING); + writeRecord(); + if (flush) { + flushAndAwait(sentinel); + assertThat(sentinel.getFlushedLogOffset()) + .isEqualTo(logTablet.localLogEndOffset()); + } + } + assertThat(release.get(5, TimeUnit.SECONDS)).isEqualTo(flush); + assertThat(sentinel.isLazyOpen()).isEqualTo(!flush); + } + + @ParameterizedTest + @ValueSource(strings = {"LAZY", "OPEN", "FAILED"}) + void testDropIsTerminal(String state) throws Exception { + if (state.equals("OPEN")) { + try (KvTablet.Guard ignored = sentinel.acquireGuard()) { + assertThat(sentinel.getKvTabletDir()).isDirectory(); + } + } else if (state.equals("FAILED")) { + openCallback = + local -> { + throw new IllegalStateException("open failed"); + }; + assertThatThrownBy(sentinel::acquireGuard) + .isInstanceOf(IllegalStateException.class); + } + sentinel.drop(); + assertGauges(0, 0, 0); + assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); + assertThat(sentinel.getRocksDBKv()).isNull(); + assertThat(sentinel.getKvTabletDir()).doesNotExist(); + assertThatThrownBy(sentinel::acquireGuard).isInstanceOf(KvStorageException.class); + } + + @Test + void testSnapshotExecutorDoesNotRunAgainstReopenedTablet() throws Exception { + Executor executor; + try (KvTablet.Guard ignored = sentinel.acquireGuard()) { + executor = sentinel.getGuardedExecutor(); + } + assertThat(sentinel.releaseKv()).isTrue(); + try (KvTablet.Guard ignored = sentinel.acquireGuard()) { + AtomicInteger executions = new AtomicInteger(); + executor.execute(executions::incrementAndGet); + assertThat(executions).hasValue(0); + sentinel.getGuardedExecutor().execute(executions::incrementAndGet); + assertThat(executions).hasValue(1); + } + } + + @Test + void testMaintenancePinsTabletUntilCompletion() throws Exception { + open(); + try (BlockingCallback blocked = new BlockingCallback()) { + FutureTask task = + start( + () -> { + sentinel.getGuardedExecutor().execute(blocked::run); + return null; + }); + blocked.awaitEntered(); + FutureTask release = start(sentinel::releaseKv); + awaitState(KvTabletLazyLifecycle.LazyState.RELEASING); + assertThat(release.isDone()).isFalse(); + blocked.close(); + task.get(5, TimeUnit.SECONDS); + assertThat(release.get(5, TimeUnit.SECONDS)).isTrue(); + assertThat(lifecycle.getActivePins()).isZero(); + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testEpochFencingRejectsStaleOpen(boolean leader) throws Exception { + AtomicInteger epoch = new AtomicInteger(1); + if (leader) { + leaderEpochSupplier = epoch::get; + } else { + bucketEpochSupplier = epoch::get; + } + try (BlockingCallback blocked = new BlockingCallback()) { + openCallback = + local -> { + blocked.run(); + return openTablet(sentinel); + }; + FutureTask task = + start( + () -> { + assertThatThrownBy(sentinel::acquireGuard) + .isInstanceOf(RuntimeException.class); + return null; + }); + blocked.awaitEntered(); + epoch.incrementAndGet(); + blocked.close(); + task.get(5, TimeUnit.SECONDS); + assertThat(lifecycle.getLazyState()) + .isEqualTo(KvTabletLazyLifecycle.LazyState.FAILED); + assertThat(sentinel.getKvTabletDir()).doesNotExist(); + } + } + + @Test + void testConcurrentRequestsShareOpenDespiteCommitCallbackFailure() throws Exception { + commitCallback = + kv -> { + throw new IllegalStateException("commit failed"); + }; + AtomicInteger opens = new AtomicInteger(); + try (BlockingCallback blocked = new BlockingCallback()) { + openCallback = + local -> { + opens.incrementAndGet(); + blocked.run(); + return openTablet(sentinel); + }; + List> requests = new java.util.ArrayList<>(); + for (int i = 0; i < 5; i++) { + requests.add( + start( + () -> { + open(); + return null; + })); + } + blocked.awaitEntered(); + blocked.close(); + for (FutureTask request : requests) { + request.get(5, TimeUnit.SECONDS); + } + assertThat(opens).hasValue(1); + } + try (KvTablet.Guard guard = sentinel.acquireGuard()) { + assertThat( + guard.getTablet() + .multiGet(Collections.singletonList("missing".getBytes()))) + .containsExactly((ByteArraySlice) null); + guard.close(); + guard.close(); + assertThat(lifecycle.getActivePins()).isZero(); + } + } + + @Test + void testReleaseDrainTimeoutRollsBackToOpen() throws Exception { + try (KvTablet.Guard ignored = sentinel.acquireGuard()) { + FutureTask release = start(sentinel::releaseKv); + awaitState(KvTabletLazyLifecycle.LazyState.RELEASING); + clock.advanceTime(1, TimeUnit.SECONDS); + assertThat(release.get(5, TimeUnit.SECONDS)).isFalse(); + assertThat(sentinel.isLazyOpen()).isTrue(); + } + assertThat(lifecycle.getActivePins()).isZero(); + } + + @Test + void testExponentialBackoffAndRecovery() throws Exception { + openCallback = + local -> { + throw new IllegalStateException("open failed"); + }; + for (int cooldown : new int[] {10, 20, 40}) { + assertThatThrownBy(sentinel::acquireGuard) + .isInstanceOf(IllegalStateException.class); + assertGauges(0, 0, 1); + clock.advanceTime(cooldown - 1, TimeUnit.SECONDS); + assertThatThrownBy(sentinel::acquireGuard) + .isInstanceOf(KvStorageException.class) + .hasMessageContaining("cooldown"); + clock.advanceTime(2, TimeUnit.SECONDS); + } + openCallback = local -> openTablet(sentinel); + try (KvTablet.Guard ignored = sentinel.acquireGuard()) { + assertThat(sentinel.getRocksDBKv()).isNotNull(); + } + } + } } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletLazyLifecycleTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletLazyLifecycleTest.java deleted file mode 100644 index fc834605f27..00000000000 --- a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletLazyLifecycleTest.java +++ /dev/null @@ -1,1482 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.server.kv; - -import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.config.Configuration; -import org.apache.fluss.config.TableConfig; -import org.apache.fluss.exception.KvStorageException; -import org.apache.fluss.metadata.KvFormat; -import org.apache.fluss.metadata.LogFormat; -import org.apache.fluss.metadata.PhysicalTablePath; -import org.apache.fluss.metadata.SchemaInfo; -import org.apache.fluss.metadata.TableBucket; -import org.apache.fluss.metadata.TablePath; -import org.apache.fluss.record.KvRecord; -import org.apache.fluss.record.KvRecordBatch; -import org.apache.fluss.record.KvRecordTestUtils; -import org.apache.fluss.record.TestData; -import org.apache.fluss.record.TestingSchemaGetter; -import org.apache.fluss.row.encode.ValueEncoder; -import org.apache.fluss.server.log.LogManager; -import org.apache.fluss.server.log.LogTablet; -import org.apache.fluss.server.metrics.group.TabletServerMetricGroup; -import org.apache.fluss.server.metrics.group.TestingMetricGroups; -import org.apache.fluss.server.storage.LocalDiskManager; -import org.apache.fluss.server.utils.ResourceGuard; -import org.apache.fluss.server.zk.NOPErrorHandler; -import org.apache.fluss.server.zk.ZooKeeperClient; -import org.apache.fluss.server.zk.ZooKeeperExtension; -import org.apache.fluss.testutils.common.AllCallbackWrapper; -import org.apache.fluss.testutils.common.CommonTestUtils; -import org.apache.fluss.types.RowType; -import org.apache.fluss.utils.ByteArraySlice; -import org.apache.fluss.utils.clock.Clock; -import org.apache.fluss.utils.clock.ManualClock; -import org.apache.fluss.utils.clock.SystemClock; -import org.apache.fluss.utils.concurrent.FlussScheduler; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - -import java.io.File; -import java.time.Duration; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executor; -import java.util.concurrent.FutureTask; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; - -import static org.apache.fluss.compression.ArrowCompressionInfo.DEFAULT_COMPRESSION; -import static org.apache.fluss.record.TestData.DATA1_SCHEMA_PK; -import static org.apache.fluss.testutils.DataTestUtils.compactedRow; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -/** Test for {@link KvTabletLazyLifecycle}. */ -final class KvTabletLazyLifecycleTest { - - @RegisterExtension - public static final AllCallbackWrapper ZOO_KEEPER_EXTENSION_WRAPPER = - new AllCallbackWrapper<>(new ZooKeeperExtension()); - - private static final short SCHEMA_ID = 1; - private static final RowType BASE_ROW_TYPE = TestData.DATA1_ROW_TYPE; - - private final KvRecordTestUtils.KvRecordBatchFactory batchFactory = - KvRecordTestUtils.KvRecordBatchFactory.of(SCHEMA_ID); - private final KvRecordTestUtils.KvRecordFactory recordFactory = - KvRecordTestUtils.KvRecordFactory.of(BASE_ROW_TYPE); - - private static ZooKeeperClient zkClient; - - private @TempDir File tempDir; - - private LocalDiskManager localDiskManager; - private LogManager logManager; - private KvManager kvManager; - private Configuration conf; - - private PhysicalTablePath physicalPath; - private TableBucket tableBucket; - private LogTablet logTablet; - - @BeforeAll - static void baseBeforeAll() { - zkClient = - ZOO_KEEPER_EXTENSION_WRAPPER - .getCustomExtension() - .getZooKeeperClient(NOPErrorHandler.INSTANCE); - } - - @BeforeEach - void setup() throws Exception { - conf = new Configuration(); - conf.setString(ConfigOptions.DATA_DIR, tempDir.getAbsolutePath()); - conf.set(ConfigOptions.KV_LAZY_OPEN_ENABLED, true); - - conf.set(ConfigOptions.TABLET_SERVER_ID, 1); - localDiskManager = LocalDiskManager.create(conf); - logManager = - LogManager.create( - conf, - zkClient, - new FlussScheduler(1), - SystemClock.getInstance(), - TestingMetricGroups.TABLET_SERVER_METRICS, - localDiskManager); - kvManager = - KvManager.create( - conf, - zkClient, - logManager, - TestingMetricGroups.TABLET_SERVER_METRICS, - localDiskManager); - kvManager.startup(); - - TablePath tablePath = TablePath.of("db1", "t_lazy"); - physicalPath = - PhysicalTablePath.of(tablePath.getDatabaseName(), tablePath.getTableName(), null); - tableBucket = new TableBucket(20001L, 0); - logTablet = - logManager.getOrCreateLog( - tempDir, physicalPath, tableBucket, LogFormat.ARROW, 1, true); - } - - @AfterEach - void tearDown() throws Exception { - if (kvManager != null) { - kvManager.shutdown(); - } - if (logManager != null) { - logManager.shutdown(); - } - if (localDiskManager != null) { - localDiskManager.close(); - } - } - - // ---- Helper methods ---- - - private KvTablet createSentinel() { - KvTablet tablet = - KvTablet.createLazySentinel( - physicalPath, - tableBucket, - logTablet, - TestingMetricGroups.TABLET_SERVER_METRICS); - kvManager.registerKv(tableBucket, tablet); - return tablet; - } - - private KvTabletLazyLifecycle configureLifecycle(KvTablet sentinel, Clock clock) { - return configureLifecycle(sentinel, clock, KvManager.RELEASE_DRAIN_TIMEOUT_MS); - } - - private KvTabletLazyLifecycle configureLifecycle( - KvTablet sentinel, Clock clock, long drainTimeout) { - KvTabletLazyLifecycle lifecycle = sentinel.getLifecycle(); - lifecycle.configureLazyOpen( - clock, - kvManager.getOpenSemaphore(), - kvManager.getOpenTimeoutMs(), - kvManager.getFailedBackoffBaseMs(), - kvManager.getFailedBackoffMaxMs(), - drainTimeout); - lifecycle.setLeaderEpochSupplier(() -> 1); - lifecycle.setBucketEpochSupplier(() -> 1); - lifecycle.setTabletDirSupplier( - () -> - kvManager.getTabletDirPath( - logTablet.getDataDir(), physicalPath, tableBucket)); - lifecycle.setOpenCallback( - hasLocal -> - kvManager.createKvTabletUnregistered( - physicalPath, - tableBucket, - logTablet, - KvFormat.COMPACTED, - new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), - new TableConfig(new Configuration()), - DEFAULT_COMPRESSION, - null)); - lifecycle.setCommitCallback(kv -> {}); - lifecycle.setReleaseCallback(kv -> {}); - lifecycle.setDropCallback(kv -> {}); - return lifecycle; - } - - private void flushAndAwait(KvTablet tablet) throws Exception { - long offset = logTablet.localLogEndOffset(); - tablet.requestFlush(offset, NOPErrorHandler.INSTANCE); - CommonTestUtils.waitUntil( - () -> tablet.getFlushedLogOffset() >= offset, - Duration.ofSeconds(10), - "KV operation did not complete"); - } - - @Test - void testMaintenanceDoesNotRefreshIdleTime() throws Exception { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - try (KvTablet.Guard ignored = sentinel.acquireGuard()) { - // Open through a business request. - } - long lastAccess = sentinel.getLastAccessTimestamp(); - clock.advanceTime(10, TimeUnit.MINUTES); - sentinel.getGuardedExecutor().execute(() -> {}); - assertThat(sentinel.getLastAccessTimestamp()).isEqualTo(lastAccess); - assertThat(lifecycle.canRelease(TimeUnit.MINUTES.toMillis(10), clock.milliseconds())) - .isTrue(); - try (KvTablet.Guard ignored = sentinel.acquireGuard()) { - assertThat(sentinel.getLastAccessTimestamp()).isEqualTo(clock.milliseconds()); - } - assertThat(lifecycle.canRelease(TimeUnit.MINUTES.toMillis(10), clock.milliseconds())) - .isFalse(); - } - - @ParameterizedTest - @ValueSource(booleans = {true, false}) - void testCloseWaitsForSlowRelease(boolean delete) throws Exception { - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = - configureLifecycle(sentinel, SystemClock.getInstance(), 20); - try (KvTablet.Guard ignored = sentinel.acquireGuard()) { - // Open an empty tablet eligible for idle release. - } - CountDownLatch entered = new CountDownLatch(1); - CountDownLatch finish = new CountDownLatch(1); - lifecycle.setReleaseCallback( - kv -> { - entered.countDown(); - try { - finish.await(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } - }); - FutureTask release = new FutureTask<>(sentinel::releaseKv); - FutureTask drop = - new FutureTask<>( - () -> { - if (delete) { - sentinel.dropKvLazy(); - } else { - sentinel.close(); - } - return null; - }); - new Thread(release, "slow-idle-release").start(); - try { - assertThat(entered.await(5, TimeUnit.SECONDS)).isTrue(); - new Thread(drop, "drop-slow-release").start(); - assertThatThrownBy(() -> drop.get(200, TimeUnit.MILLISECONDS)) - .isInstanceOf(TimeoutException.class); - } finally { - finish.countDown(); - release.get(5, TimeUnit.SECONDS); - drop.get(5, TimeUnit.SECONDS); - } - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); - assertThat(sentinel.getRocksDBKv()).isNull(); - assertThat(sentinel.getKvTabletDir().exists()).isEqualTo(!delete); - assertThatThrownBy(sentinel::acquireGuard).isInstanceOf(KvStorageException.class); - } - - @ParameterizedTest - @ValueSource(booleans = {true, false}) - void testCloseWaitsForFencedOpenCleanup(boolean shutdown) throws Exception { - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = - configureLifecycle(sentinel, SystemClock.getInstance(), 20); - CountDownLatch prepared = new CountDownLatch(1); - CountDownLatch returnCandidate = new CountDownLatch(1); - AtomicReference candidate = new AtomicReference<>(); - AtomicReference lease = new AtomicReference<>(); - lifecycle.setOpenCallback( - hasLocal -> { - KvTablet opened = - kvManager.createKvTabletUnregistered( - physicalPath, - tableBucket, - logTablet, - KvFormat.COMPACTED, - new TestingSchemaGetter( - new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), - new TableConfig(new Configuration()), - DEFAULT_COMPRESSION, - null); - candidate.set(opened); - lease.set(opened.getRocksDBKv().getResourceGuard().acquireResource()); - prepared.countDown(); - returnCandidate.await(); - return opened; - }); - FutureTask open = - new FutureTask<>( - () -> { - assertThatThrownBy(sentinel::acquireGuard) - .isInstanceOf(RuntimeException.class); - return null; - }); - FutureTask close = - new FutureTask<>( - () -> { - if (shutdown) { - kvManager.shutdown(); - } else { - kvManager.dropKv(tableBucket); - } - return null; - }); - new Thread(open, "fenced-open").start(); - try { - assertThat(prepared.await(5, TimeUnit.SECONDS)).isTrue(); - new Thread(close, "close-fenced-open").start(); - CommonTestUtils.waitUntil( - () -> lifecycle.getLazyState() == KvTabletLazyLifecycle.LazyState.CLOSING, - Duration.ofSeconds(5), - "Close did not fence opening tablet"); - returnCandidate.countDown(); - CommonTestUtils.waitUntil( - () -> candidate.get().getRocksDBKv().getResourceGuard().isClosed(), - Duration.ofSeconds(5), - "Candidate cleanup did not start"); - assertThatThrownBy(() -> close.get(200, TimeUnit.MILLISECONDS)) - .isInstanceOf(TimeoutException.class); - assertThat(kvManager.getKv(tableBucket)).contains(sentinel); - assertThatThrownBy(() -> kvManager.registerKv(tableBucket, sentinel)) - .isInstanceOf(IllegalStateException.class); - } finally { - returnCandidate.countDown(); - if (lease.get() != null) { - lease.get().close(); - } - open.get(5, TimeUnit.SECONDS); - close.get(5, TimeUnit.SECONDS); - } - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); - assertThat(sentinel.getRocksDBKv()).isNull(); - assertThat(sentinel.getKvTabletDir()).doesNotExist(); - if (shutdown) { - kvManager = null; - } else { - assertThat(kvManager.getKv(tableBucket)).isEmpty(); - KvTablet replacement = createSentinel(); - configureLifecycle(replacement, SystemClock.getInstance()); - try (KvTablet.Guard ignored = replacement.acquireGuard()) { - assertThat(replacement.getKvTabletDir()).isDirectory(); - } - } - } - - @Test - void testDropAfterCloseDeletesLocalDataOnlyOnce() throws Exception { - KvTablet sentinel = createSentinel(); - configureLifecycle(sentinel, SystemClock.getInstance()); - try (KvTablet.Guard ignored = sentinel.acquireGuard()) { - assertThat(sentinel.getKvTabletDir()).isDirectory(); - } - sentinel.close(); - assertThat(sentinel.getKvTabletDir()).isDirectory(); - sentinel.dropKvLazy(); - assertThat(sentinel.getKvTabletDir()).doesNotExist(); - kvManager.unregisterKv(tableBucket, sentinel); - KvTablet replacement = createSentinel(); - configureLifecycle(replacement, SystemClock.getInstance()); - try (KvTablet.Guard ignored = replacement.acquireGuard()) { - sentinel.dropKvLazy(); - assertThat(replacement.getKvTabletDir()).isDirectory(); - } - } - - // ---- Tests ---- - - @Test - void testAcquireGuardTriggersOpen() throws Exception { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); - - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - // Verify we can use the tablet inside the guard - List values = - sentinel.multiGet(Collections.singletonList("k1".getBytes())); - assertThat(values).hasSize(1); - assertThat(values.get(0)).isNull(); - } - } - - @Test - void testAcquireGuardFastPath() throws Exception { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - // Track open callback invocations to prove fast path skips it - AtomicInteger openCount = new AtomicInteger(0); - lifecycle.setOpenCallback( - hasLocal -> { - openCount.incrementAndGet(); - return kvManager.createKvTabletUnregistered( - physicalPath, - tableBucket, - logTablet, - KvFormat.COMPACTED, - new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), - new TableConfig(new Configuration()), - DEFAULT_COMPRESSION, - null); - }); - - // First open via slow path - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - } - assertThat(openCount.get()).isEqualTo(1); - - // Second acquireGuard should take the fast path (already OPEN, no openCallback) - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - List values = - sentinel.multiGet(Collections.singletonList("nonexistent".getBytes())); - assertThat(values).hasSize(1); - assertThat(values.get(0)).isNull(); - } - // openCallback must NOT have been called again - assertThat(openCount.get()).isEqualTo(1); - } - - @Test - void testFailedBackoffRejectsOpen() { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - // Override openCallback to throw - lifecycle.setOpenCallback( - hasLocal -> { - throw new RuntimeException("simulated open failure"); - }); - - // First acquireGuard should fail - assertThatThrownBy(sentinel::acquireGuard).isInstanceOf(RuntimeException.class); - - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.FAILED); - - // Subsequent acquireGuard within cooldown should throw with cooldown message - assertThatThrownBy(sentinel::acquireGuard) - .isInstanceOf(KvStorageException.class) - .hasMessageContaining("cooldown"); - } - - @Test - void testFailedBackoffRetryAfterCooldown() throws Exception { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - // Override openCallback to throw - lifecycle.setOpenCallback( - hasLocal -> { - throw new RuntimeException("simulated open failure"); - }); - - // First acquireGuard fails - assertThatThrownBy(sentinel::acquireGuard).isInstanceOf(RuntimeException.class); - - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.FAILED); - - // Advance clock past the backoff (5000 * 2^1 = 10000ms for failureCount=1) - clock.advanceTime(11, TimeUnit.SECONDS); - - // Fix the openCallback - lifecycle.setOpenCallback( - hasLocal -> - kvManager.createKvTabletUnregistered( - physicalPath, - tableBucket, - logTablet, - KvFormat.COMPACTED, - new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), - new TableConfig(new Configuration()), - DEFAULT_COMPRESSION, - null)); - - // Now acquireGuard should succeed - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - } - } - - @Test - void testReleaseKvSuccess() throws Exception { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - // Open and write+flush data - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - KvRecord record = recordFactory.ofRecord("k1".getBytes(), new Object[] {1, "a"}); - KvRecordBatch batch = batchFactory.ofRecords(Collections.singletonList(record)); - sentinel.putAsLeader(batch, null); - flushAndAwait(sentinel); - } - - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - - // Capture flushed offset before release (detach may reset it) - long flushedOffsetBeforeRelease = sentinel.getFlushedLogOffset(); - - // Release should succeed - boolean released = sentinel.releaseKv(); - assertThat(released).isTrue(); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); - // Row count and flushed offset should be cached - assertThat(lifecycle.getCachedRowCount()).isEqualTo(1L); - assertThat(lifecycle.getCachedFlushedLogOffset()).isEqualTo(flushedOffsetBeforeRelease); - } - - @Test - void testReleaseKvRejectsUnflushedData() throws Exception { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - // Open and write data WITHOUT flush - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - KvRecord record = recordFactory.ofRecord("k1".getBytes(), new Object[] {1, "a"}); - KvRecordBatch batch = batchFactory.ofRecords(Collections.singletonList(record)); - sentinel.putAsLeader(batch, null); - // Intentionally NOT flushing - } - - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - - // Release should fail because data is unflushed - boolean released = sentinel.releaseKv(); - assertThat(released).isFalse(); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - } - - @Test - void testReleaseAndReopenPreservesData() throws Exception { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - // Open, write+flush - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - KvRecord record = recordFactory.ofRecord("k1".getBytes(), new Object[] {1, "a"}); - KvRecordBatch batch = batchFactory.ofRecords(Collections.singletonList(record)); - sentinel.putAsLeader(batch, null); - flushAndAwait(sentinel); - } - - // Release to LAZY - boolean released = sentinel.releaseKv(); - assertThat(released).isTrue(); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); - - // Reopen via acquireGuard and verify data is still readable - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - List values = - sentinel.multiGet(Collections.singletonList("k1".getBytes())); - assertThat(values).hasSize(1); - byte[] expected = - ValueEncoder.encodeValue( - SCHEMA_ID, compactedRow(BASE_ROW_TYPE, new Object[] {1, "a"})); - assertThat(values.get(0).toByteArray()).isEqualTo(expected); - } - } - - @Test - void testFlushRequestWhileReleaseDrainsWriter() throws Exception { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - FutureTask releaseTask = new FutureTask<>(sentinel::releaseKv); - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - new Thread(releaseTask, "release-during-flush-request").start(); - CommonTestUtils.waitUntil( - () -> lifecycle.getLazyState() == KvTabletLazyLifecycle.LazyState.RELEASING, - Duration.ofSeconds(5), - "Timed out waiting for RELEASING state"); - KvRecord record = recordFactory.ofRecord("k1".getBytes(), new Object[] {1, "a"}); - sentinel.putAsLeader(batchFactory.ofRecords(Collections.singletonList(record)), null); - flushAndAwait(sentinel); - assertThat(sentinel.getFlushedLogOffset()).isEqualTo(logTablet.localLogEndOffset()); - } - releaseTask.get(5, TimeUnit.SECONDS); - } - - @Test - void testReleaseKvRejectsWriteThatCompletesDuringDrain() throws Exception { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - KvTablet.Guard guard = sentinel.acquireGuard(); - FutureTask releaseTask = new FutureTask<>(sentinel::releaseKv); - Thread releaseThread = new Thread(releaseTask, "release-during-write"); - releaseThread.start(); - - CommonTestUtils.waitUntil( - () -> lifecycle.getLazyState() == KvTabletLazyLifecycle.LazyState.RELEASING, - Duration.ofSeconds(5), - "Timed out waiting for RELEASING state"); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.RELEASING); - - KvRecord record = recordFactory.ofRecord("k1".getBytes(), new Object[] {1, "a"}); - KvRecordBatch batch = batchFactory.ofRecords(Collections.singletonList(record)); - sentinel.putAsLeader(batch, null); - guard.close(); - assertThat(releaseTask.get(5, TimeUnit.SECONDS)).isFalse(); - - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - assertThat(sentinel.getKvPreWriteBuffer().pendingFlushBytes()).isPositive(); - } - - @Test - void testDropKvFromLazy() { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); - - sentinel.dropKvLazy(); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); - } - - @Test - void testDropKvFromOpen() throws Exception { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - // Open the tablet - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - } - - File kvDir = sentinel.getKvTabletDir(); - assertThat(kvDir).isNotNull(); - assertThat(kvDir).exists(); - - sentinel.dropKvLazy(); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); - assertThat(sentinel.getRocksDBKv()).isNull(); - assertThat(kvDir).doesNotExist(); - } - - @Test - void testDropKvDuringReleasingRollbackStillClosesAndDeletes() throws Exception { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - KvRecord record = recordFactory.ofRecord("k1".getBytes(), new Object[] {1, "a"}); - KvRecordBatch batch = batchFactory.ofRecords(Collections.singletonList(record)); - sentinel.putAsLeader(batch, null); - flushAndAwait(sentinel); - } - - File kvDir = sentinel.getKvTabletDir(); - assertThat(kvDir).isNotNull(); - assertThat(kvDir).exists(); - - CountDownLatch releaseCallbackEntered = new CountDownLatch(1); - CountDownLatch allowReleaseCallbackFinish = new CountDownLatch(1); - lifecycle.setReleaseCallback( - kv -> { - releaseCallbackEntered.countDown(); - try { - allowReleaseCallbackFinish.await(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - throw new RuntimeException("simulated release failure"); - }); - - FutureTask releaseTask = new FutureTask<>(sentinel::releaseKv); - Thread releaseThread = new Thread(releaseTask, "release-with-rollback"); - releaseThread.start(); - - assertThat(releaseCallbackEntered.await(5, TimeUnit.SECONDS)).isTrue(); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.RELEASING); - - FutureTask dropTask = - new FutureTask<>( - () -> { - sentinel.dropKvLazy(); - return null; - }); - Thread dropThread = new Thread(dropTask, "drop-during-release-rollback"); - dropThread.start(); - - allowReleaseCallbackFinish.countDown(); - assertThat(releaseTask.get(5, TimeUnit.SECONDS)).isFalse(); - dropTask.get(5, TimeUnit.SECONDS); - - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); - assertThat(sentinel.getRocksDBKv()).isNull(); - assertThat(kvDir).doesNotExist(); - } - - @Test - void testCommitCallbackFailureDoesNotFailOpen() throws Exception { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - lifecycle.setCommitCallback( - kv -> { - throw new RuntimeException("simulated commit callback failure"); - }); - - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - assertThat(sentinel.getRocksDBKv()).isNotNull(); - } - } - - @Test - void testDropWaitsForCommitCallbackCompletion() throws Exception { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - CountDownLatch commitCallbackEntered = new CountDownLatch(1); - CountDownLatch allowCommitCallbackFinish = new CountDownLatch(1); - lifecycle.setCommitCallback( - kv -> { - commitCallbackEntered.countDown(); - try { - allowCommitCallbackFinish.await(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - }); - - AtomicReference openFailure = new AtomicReference<>(); - Thread openThread = - new Thread( - () -> { - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - // no-op - } catch (Throwable t) { - openFailure.set(t); - } - }, - "open-with-blocked-commit"); - openThread.start(); - - assertThat(commitCallbackEntered.await(5, TimeUnit.SECONDS)).isTrue(); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - - FutureTask dropTask = - new FutureTask<>( - () -> { - sentinel.dropKvLazy(); - return null; - }); - Thread dropThread = new Thread(dropTask, "drop-during-commit"); - dropThread.start(); - - // Wait for drop thread to block (commit callback holds the lock) - CommonTestUtils.waitUntil( - () -> { - Thread.State s = dropThread.getState(); - return s == Thread.State.WAITING || s == Thread.State.TIMED_WAITING; - }, - Duration.ofSeconds(5), - "Timed out waiting for drop thread to block"); - assertThat(dropTask.isDone()).isFalse(); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSING); - - allowCommitCallbackFinish.countDown(); - openThread.join(TimeUnit.SECONDS.toMillis(5)); - assertThat(openThread.isAlive()).isFalse(); - dropTask.get(5, TimeUnit.SECONDS); - - // The open thread may or may not see a KvStorageException depending on the race - // between guard.close() and dropKvLazy() transitioning to CLOSED. Both outcomes are valid: - // - null: guard closed before drop detached RocksDB - // - KvStorageException: drop completed while open thread was still in acquireGuard - Throwable failure = openFailure.get(); - assertThat(failure == null || failure instanceof KvStorageException) - .as( - "Expected null or KvStorageException but got: %s", - failure != null ? failure.getClass().getName() : "null") - .isTrue(); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); - assertThat(sentinel.getRocksDBKv()).isNull(); - } - - @Test - void testCloseMarksLazyTabletClosed() throws Exception { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - } - - sentinel.close(); - - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); - assertThat(sentinel.getRocksDBKv()).isNull(); - assertThatThrownBy(sentinel::acquireGuard).isInstanceOf(KvStorageException.class); - } - - @Test - void testSnapshotExecutorDoesNotRunAgainstReopenedTablet() throws Exception { - KvTablet sentinel = createSentinel(); - configureLifecycle(sentinel, new ManualClock(System.currentTimeMillis())); - Executor executor; - try (KvTablet.Guard ignored = sentinel.acquireGuard()) { - executor = sentinel.getGuardedExecutor(); - } - assertThat(sentinel.releaseKv()).isTrue(); - try (KvTablet.Guard ignored = sentinel.acquireGuard()) { - AtomicInteger executions = new AtomicInteger(); - executor.execute(executions::incrementAndGet); - assertThat(executions).hasValue(0); - sentinel.getGuardedExecutor().execute(executions::incrementAndGet); - assertThat(executions).hasValue(1); - } - } - - @Test - void testGuardedExecutorPinsOpenTabletUntilTaskCompletes() throws Exception { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - } - - CountDownLatch taskStarted = new CountDownLatch(1); - CountDownLatch allowTaskComplete = new CountDownLatch(1); - FutureTask guardedTask = - new FutureTask<>( - () -> { - sentinel.getGuardedExecutor() - .execute( - () -> { - taskStarted.countDown(); - try { - allowTaskComplete.await(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - }); - return null; - }); - Thread taskThread = new Thread(guardedTask, "guarded-executor-task"); - taskThread.start(); - - assertThat(taskStarted.await(5, TimeUnit.SECONDS)).isTrue(); - assertThat(sentinel.getActivePins()).isEqualTo(1); - - FutureTask releaseTask = new FutureTask<>(sentinel::releaseKv); - Thread releaseThread = new Thread(releaseTask, "release-task"); - releaseThread.start(); - - CommonTestUtils.waitUntil( - () -> lifecycle.getLazyState() == KvTabletLazyLifecycle.LazyState.RELEASING, - Duration.ofSeconds(5), - "Timed out waiting for RELEASING state"); - assertThat(releaseTask.isDone()).isFalse(); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.RELEASING); - - allowTaskComplete.countDown(); - assertThat(releaseTask.get(5, TimeUnit.SECONDS)).isTrue(); - taskThread.join(TimeUnit.SECONDS.toMillis(5)); - assertThat(taskThread.isAlive()).isFalse(); - - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); - assertThat(sentinel.getActivePins()).isZero(); - } - - @Test - void testGaugeUpdatesOnTransitions() throws Exception { - TabletServerMetricGroup mg = TestingMetricGroups.TABLET_SERVER_METRICS; - // Record baseline counts before this test - int baseLazy = mg.kvTabletLazyCount().get(); - int baseOpen = mg.kvTabletOpenCount().get(); - int baseFailed = mg.kvTabletFailedCount().get(); - - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - // After configureLazyOpen: LAZY +1 - assertThat(mg.kvTabletLazyCount().get() - baseLazy).isEqualTo(1); - assertThat(mg.kvTabletOpenCount().get() - baseOpen).isEqualTo(0); - assertThat(mg.kvTabletFailedCount().get() - baseFailed).isEqualTo(0); - - // Open: LAZY -1, OPEN +1 - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - KvRecord record = recordFactory.ofRecord("k1".getBytes(), new Object[] {1, "a"}); - KvRecordBatch batch = batchFactory.ofRecords(Collections.singletonList(record)); - sentinel.putAsLeader(batch, null); - flushAndAwait(sentinel); - } - assertThat(mg.kvTabletLazyCount().get() - baseLazy).isEqualTo(0); - assertThat(mg.kvTabletOpenCount().get() - baseOpen).isEqualTo(1); - - // Release: OPEN -1, LAZY +1 - boolean released = sentinel.releaseKv(); - assertThat(released).isTrue(); - assertThat(mg.kvTabletLazyCount().get() - baseLazy).isEqualTo(1); - assertThat(mg.kvTabletOpenCount().get() - baseOpen).isEqualTo(0); - - // Open again: LAZY -1, OPEN +1 - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - } - assertThat(mg.kvTabletLazyCount().get() - baseLazy).isEqualTo(0); - assertThat(mg.kvTabletOpenCount().get() - baseOpen).isEqualTo(1); - - // Drop: OPEN -1 - sentinel.dropKvLazy(); - assertThat(mg.kvTabletOpenCount().get() - baseOpen).isEqualTo(0); - assertThat(mg.kvTabletLazyCount().get() - baseLazy).isEqualTo(0); - assertThat(mg.kvTabletFailedCount().get() - baseFailed).isEqualTo(0); - } - - @Test - void testEpochFencingRejectsStaleOpenOnLeaderEpochChange() throws Exception { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - AtomicInteger leaderEpoch = new AtomicInteger(1); - lifecycle.setLeaderEpochSupplier(leaderEpoch::get); - - CountDownLatch openStarted = new CountDownLatch(1); - CountDownLatch allowOpenFinish = new CountDownLatch(1); - - lifecycle.setOpenCallback( - hasLocal -> { - openStarted.countDown(); - try { - allowOpenFinish.await(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - return kvManager.createKvTabletUnregistered( - physicalPath, - tableBucket, - logTablet, - KvFormat.COMPACTED, - new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), - new TableConfig(new Configuration()), - DEFAULT_COMPRESSION, - null); - }); - - AtomicReference openFailure = new AtomicReference<>(); - Thread openThread = - new Thread( - () -> { - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - // no-op - } catch (Throwable t) { - openFailure.set(t); - } - }, - "epoch-fence-leader"); - openThread.start(); - - assertThat(openStarted.await(5, TimeUnit.SECONDS)).isTrue(); - - // Change leader epoch while open is in progress - leaderEpoch.set(2); - allowOpenFinish.countDown(); - - openThread.join(TimeUnit.SECONDS.toMillis(5)); - assertThat(openThread.isAlive()).isFalse(); - - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.FAILED); - assertThat(openFailure.get()).isNotNull(); - } - - @Test - void testEpochFencingRejectsStaleOpenOnBucketEpochChange() throws Exception { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - AtomicInteger bucketEpoch = new AtomicInteger(1); - lifecycle.setBucketEpochSupplier(bucketEpoch::get); - - CountDownLatch openStarted = new CountDownLatch(1); - CountDownLatch allowOpenFinish = new CountDownLatch(1); - - lifecycle.setOpenCallback( - hasLocal -> { - openStarted.countDown(); - try { - allowOpenFinish.await(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - return kvManager.createKvTabletUnregistered( - physicalPath, - tableBucket, - logTablet, - KvFormat.COMPACTED, - new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), - new TableConfig(new Configuration()), - DEFAULT_COMPRESSION, - null); - }); - - AtomicReference openFailure = new AtomicReference<>(); - Thread openThread = - new Thread( - () -> { - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - // no-op - } catch (Throwable t) { - openFailure.set(t); - } - }, - "epoch-fence-bucket"); - openThread.start(); - - assertThat(openStarted.await(5, TimeUnit.SECONDS)).isTrue(); - - // Change bucket epoch while open is in progress - bucketEpoch.set(2); - allowOpenFinish.countDown(); - - openThread.join(TimeUnit.SECONDS.toMillis(5)); - assertThat(openThread.isAlive()).isFalse(); - - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.FAILED); - assertThat(openFailure.get()).isNotNull(); - } - - @Test - void testConcurrentAcquireGuardOnlyOpensOnce() throws Exception { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - AtomicInteger openCount = new AtomicInteger(0); - CountDownLatch openStarted = new CountDownLatch(1); - CountDownLatch allowOpenFinish = new CountDownLatch(1); - - lifecycle.setOpenCallback( - hasLocal -> { - openCount.incrementAndGet(); - openStarted.countDown(); - try { - allowOpenFinish.await(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - return kvManager.createKvTabletUnregistered( - physicalPath, - tableBucket, - logTablet, - KvFormat.COMPACTED, - new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), - new TableConfig(new Configuration()), - DEFAULT_COMPRESSION, - null); - }); - - int threadCount = 5; - CountDownLatch barrier = new CountDownLatch(1); - AtomicInteger successCount = new AtomicInteger(0); - AtomicInteger failureCount = new AtomicInteger(0); - Thread[] threads = new Thread[threadCount]; - - for (int i = 0; i < threadCount; i++) { - threads[i] = - new Thread( - () -> { - try { - barrier.await(5, TimeUnit.SECONDS); - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - successCount.incrementAndGet(); - } - } catch (Exception e) { - failureCount.incrementAndGet(); - } - }, - "concurrent-acquire-" + i); - threads[i].start(); - } - - // Release all threads at once - barrier.countDown(); - - // Wait for the opener thread to enter doSlowOpen - assertThat(openStarted.await(5, TimeUnit.SECONDS)).isTrue(); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPENING); - - // Let the open complete - allowOpenFinish.countDown(); - - // Wait for all threads to finish - for (Thread t : threads) { - t.join(TimeUnit.SECONDS.toMillis(10)); - assertThat(t.isAlive()).isFalse(); - } - - assertThat(openCount.get()).isEqualTo(1); - assertThat(successCount.get()).isEqualTo(threadCount); - assertThat(failureCount.get()).isZero(); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - } - - @Test - void testDropKvDuringOpeningFencesAndCloses() throws Exception { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - CountDownLatch openStarted = new CountDownLatch(1); - CountDownLatch allowOpenFinish = new CountDownLatch(1); - - lifecycle.setOpenCallback( - hasLocal -> { - openStarted.countDown(); - try { - allowOpenFinish.await(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - return kvManager.createKvTabletUnregistered( - physicalPath, - tableBucket, - logTablet, - KvFormat.COMPACTED, - new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), - new TableConfig(new Configuration()), - DEFAULT_COMPRESSION, - null); - }); - - AtomicReference openFailure = new AtomicReference<>(); - Thread openThread = - new Thread( - () -> { - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - // no-op - } catch (Throwable t) { - openFailure.set(t); - } - }, - "open-for-drop-fence"); - openThread.start(); - - assertThat(openStarted.await(5, TimeUnit.SECONDS)).isTrue(); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPENING); - - FutureTask dropTask = - new FutureTask<>( - () -> { - sentinel.dropKvLazy(); - return null; - }); - Thread dropThread = new Thread(dropTask, "drop-during-opening"); - dropThread.start(); - - // Let the open finish — commitOpenResult should see generation mismatch - allowOpenFinish.countDown(); - - openThread.join(TimeUnit.SECONDS.toMillis(5)); - assertThat(openThread.isAlive()).isFalse(); - dropTask.get(5, TimeUnit.SECONDS); - - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); - assertThat(sentinel.getRocksDBKv()).isNull(); - assertThat(openFailure.get()).isNotNull(); - } - - @Test - void testReleaseDrainTimeoutRollsBackToOpen() throws Exception { - KvTablet sentinel = createSentinel(); - - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTabletLazyLifecycle lifecycle = sentinel.getLifecycle(); - lifecycle.configureLazyOpen( - clock, - kvManager.getOpenSemaphore(), - kvManager.getOpenTimeoutMs(), - kvManager.getFailedBackoffBaseMs(), - kvManager.getFailedBackoffMaxMs(), - 100L); // 100ms drain timeout - lifecycle.setLeaderEpochSupplier(() -> 1); - lifecycle.setBucketEpochSupplier(() -> 1); - lifecycle.setTabletDirSupplier( - () -> - kvManager.getTabletDirPath( - logTablet.getDataDir(), physicalPath, tableBucket)); - lifecycle.setOpenCallback( - hasLocal -> - kvManager.createKvTabletUnregistered( - physicalPath, - tableBucket, - logTablet, - KvFormat.COMPACTED, - new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), - new TableConfig(new Configuration()), - DEFAULT_COMPRESSION, - null)); - lifecycle.setCommitCallback(kv -> {}); - lifecycle.setReleaseCallback(kv -> {}); - lifecycle.setDropCallback(kv -> {}); - - // Acquire a guard and hold it to prevent drain from completing - KvTablet.Guard guard = sentinel.acquireGuard(); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - - // Start releaseKv in a background thread — it will block waiting for drain - FutureTask releaseTask = new FutureTask<>(sentinel::releaseKv); - Thread releaseThread = new Thread(releaseTask, "release-drain-timeout"); - releaseThread.start(); - - // Wait for the release thread to enter RELEASING state - CommonTestUtils.waitUntil( - () -> lifecycle.getLazyState() == KvTabletLazyLifecycle.LazyState.RELEASING, - Duration.ofSeconds(5), - "Timed out waiting for RELEASING state"); - - // Advance clock past drain timeout to trigger rollback deterministically - clock.advanceTime(200, TimeUnit.MILLISECONDS); - - // The release should timeout and roll back to OPEN - boolean released = releaseTask.get(5, TimeUnit.SECONDS); - assertThat(released).isFalse(); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - - // Close the guard and verify pins are drained - guard.close(); - assertThat(sentinel.getActivePins()).isZero(); - } - - @Test - void testReleaseCallbackFailureRollsBackToOpenAndTabletRemainsFunctional() throws Exception { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - // Open and write+flush data - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - KvRecord record = recordFactory.ofRecord("k1".getBytes(), new Object[] {1, "a"}); - KvRecordBatch batch = batchFactory.ofRecords(Collections.singletonList(record)); - sentinel.putAsLeader(batch, null); - flushAndAwait(sentinel); - } - - // Set release callback to throw - lifecycle.setReleaseCallback( - kv -> { - throw new RuntimeException("simulated release callback failure"); - }); - - // releaseKv should return false due to callback failure - boolean released = sentinel.releaseKv(); - assertThat(released).isFalse(); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - - // Verify tablet is still functional — acquireGuard and multiGet should work - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - List values = - sentinel.multiGet(Collections.singletonList("k1".getBytes())); - assertThat(values).hasSize(1); - byte[] expected = - ValueEncoder.encodeValue( - SCHEMA_ID, compactedRow(BASE_ROW_TYPE, new Object[] {1, "a"})); - assertThat(values.get(0).toByteArray()).isEqualTo(expected); - } - - // Fix the callback and verify subsequent release succeeds - lifecycle.setReleaseCallback(kv -> {}); - released = sentinel.releaseKv(); - assertThat(released).isTrue(); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.LAZY); - } - - @Test - void testDropKvFromFailed() { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - // Override openCallback to throw - lifecycle.setOpenCallback( - hasLocal -> { - throw new RuntimeException("simulated open failure"); - }); - - // First acquireGuard should fail - assertThatThrownBy(sentinel::acquireGuard).isInstanceOf(RuntimeException.class); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.FAILED); - - // Drop from FAILED state - sentinel.dropKvLazy(); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); - } - - @Test - void testExponentialBackoffEscalatesAcrossMultipleFailures() throws Exception { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - // Override openCallback to throw - lifecycle.setOpenCallback( - hasLocal -> { - throw new RuntimeException("simulated open failure"); - }); - - // Failure 1: failureCount becomes 1, cooldown = 5000 * 2^1 = 10000ms - assertThatThrownBy(sentinel::acquireGuard).isInstanceOf(RuntimeException.class); - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.FAILED); - - // Advance 9s — still within 10s cooldown - clock.advanceTime(9, TimeUnit.SECONDS); - assertThatThrownBy(sentinel::acquireGuard) - .isInstanceOf(KvStorageException.class) - .hasMessageContaining("cooldown"); - - // Advance 2s more (total 11s, past 10s cooldown) — retry and fail again - clock.advanceTime(2, TimeUnit.SECONDS); - assertThatThrownBy(sentinel::acquireGuard).isInstanceOf(RuntimeException.class); - - // Failure 2: failureCount becomes 2, cooldown = 5000 * 2^2 = 20000ms - // Advance 19s — still within 20s cooldown - clock.advanceTime(19, TimeUnit.SECONDS); - assertThatThrownBy(sentinel::acquireGuard) - .isInstanceOf(KvStorageException.class) - .hasMessageContaining("cooldown"); - - // Advance 2s more (total 21s, past 20s cooldown) — retry and fail again - clock.advanceTime(2, TimeUnit.SECONDS); - assertThatThrownBy(sentinel::acquireGuard).isInstanceOf(RuntimeException.class); - - // Failure 3: failureCount becomes 3, cooldown = 5000 * 2^3 = 40000ms - // Advance 39s — still within 40s cooldown - clock.advanceTime(39, TimeUnit.SECONDS); - assertThatThrownBy(sentinel::acquireGuard) - .isInstanceOf(KvStorageException.class) - .hasMessageContaining("cooldown"); - - // Advance 2s more (total 41s, past 40s cooldown) — fix callback and verify recovery - clock.advanceTime(2, TimeUnit.SECONDS); - lifecycle.setOpenCallback( - hasLocal -> - kvManager.createKvTabletUnregistered( - physicalPath, - tableBucket, - logTablet, - KvFormat.COMPACTED, - new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA_PK, SCHEMA_ID)), - new TableConfig(new Configuration()), - DEFAULT_COMPRESSION, - null)); - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - assertThat(lifecycle.getLazyState()).isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - } - } - - @Test - void testGuardDoubleCloseDoesNotDecrementPinsBelowZero() { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - configureLifecycle(sentinel, clock); - - KvTablet.Guard guard = sentinel.acquireGuard(); - assertThat(sentinel.getActivePins()).isEqualTo(1); - - guard.close(); - assertThat(sentinel.getActivePins()).isZero(); - - // Second close should be idempotent — pins must not go negative - guard.close(); - assertThat(sentinel.getActivePins()).isZero(); - } - - @Test - void testAcquireGuardOnClosedTabletThrows() { - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - configureLifecycle(sentinel, clock); - - // Open the tablet - try (KvTablet.Guard guard = sentinel.acquireGuard()) { - assertThat(sentinel.getLifecycle().getLazyState()) - .isEqualTo(KvTabletLazyLifecycle.LazyState.OPEN); - } - - // Drop transitions to CLOSED - sentinel.dropKvLazy(); - assertThat(sentinel.getLifecycle().getLazyState()) - .isEqualTo(KvTabletLazyLifecycle.LazyState.CLOSED); - - // acquireGuard on a CLOSED tablet must throw - assertThatThrownBy(sentinel::acquireGuard) - .isInstanceOf(KvStorageException.class) - .hasMessageContaining("closed"); - } - - @Test - void testGaugeUpdatesOnFailedTransition() { - TabletServerMetricGroup mg = TestingMetricGroups.TABLET_SERVER_METRICS; - int baseLazy = mg.kvTabletLazyCount().get(); - int baseOpen = mg.kvTabletOpenCount().get(); - int baseFailed = mg.kvTabletFailedCount().get(); - - ManualClock clock = new ManualClock(System.currentTimeMillis()); - KvTablet sentinel = createSentinel(); - KvTabletLazyLifecycle lifecycle = configureLifecycle(sentinel, clock); - - // After configureLazyOpen: LAZY +1 - assertThat(mg.kvTabletLazyCount().get() - baseLazy).isEqualTo(1); - - // Override openCallback to throw - lifecycle.setOpenCallback( - hasLocal -> { - throw new RuntimeException("simulated open failure"); - }); - - // acquireGuard should fail and transition to FAILED - assertThatThrownBy(sentinel::acquireGuard).isInstanceOf(RuntimeException.class); - - // LAZY back to baseline, FAILED +1, OPEN unchanged - assertThat(mg.kvTabletLazyCount().get() - baseLazy).isEqualTo(0); - assertThat(mg.kvTabletFailedCount().get() - baseFailed).isEqualTo(1); - assertThat(mg.kvTabletOpenCount().get() - baseOpen).isEqualTo(0); - - // Drop from FAILED state - sentinel.dropKvLazy(); - - // All gauges back to baseline - assertThat(mg.kvTabletLazyCount().get() - baseLazy).isEqualTo(0); - assertThat(mg.kvTabletOpenCount().get() - baseOpen).isEqualTo(0); - assertThat(mg.kvTabletFailedCount().get() - baseFailed).isEqualTo(0); - } -} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaLazyOpenTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaLazyOpenTest.java index daac035e4d7..c7f7368700e 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaLazyOpenTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaLazyOpenTest.java @@ -19,14 +19,11 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; -import org.apache.fluss.fs.FsPath; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.record.ChangeType; import org.apache.fluss.record.DefaultValueRecordBatch; -import org.apache.fluss.record.KvRecord; import org.apache.fluss.record.KvRecordBatch; -import org.apache.fluss.record.KvRecordTestUtils; import org.apache.fluss.row.encode.CompactedKeyEncoder; import org.apache.fluss.row.encode.ValueEncoder; import org.apache.fluss.rpc.protocol.MergeMode; @@ -34,9 +31,6 @@ import org.apache.fluss.server.kv.KvTablet; import org.apache.fluss.server.kv.scan.ScannerContext; import org.apache.fluss.server.kv.snapshot.CompletedSnapshot; -import org.apache.fluss.server.kv.snapshot.KvFileHandle; -import org.apache.fluss.server.kv.snapshot.KvFileHandleAndLocalPath; -import org.apache.fluss.server.kv.snapshot.KvSnapshotHandle; import org.apache.fluss.server.log.LogAppendInfo; import org.apache.fluss.server.zk.NOPErrorHandler; import org.apache.fluss.server.zk.data.LeaderAndIsr; @@ -44,6 +38,8 @@ import org.apache.fluss.utils.ByteArraySlice; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.io.File; import java.time.Duration; @@ -61,6 +57,7 @@ import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID; import static org.apache.fluss.testutils.DataTestUtils.compactedRow; import static org.apache.fluss.testutils.DataTestUtils.createBasicMemoryLogRecords; +import static org.apache.fluss.testutils.DataTestUtils.genKvRecordBatch; import static org.apache.fluss.utils.FileUtils.deleteDirectory; import static org.apache.fluss.utils.Preconditions.checkNotNull; import static org.assertj.core.api.Assertions.assertThat; @@ -81,14 +78,11 @@ void testPeriodicSnapshotDoesNotKeepIdleTabletOpen() throws Exception { Replica replica = makeLazyLeaderReplica(); replica.lookups(Collections.singletonList(key(1))); KvTablet tablet = checkNotNull(replica.getKvTablet()); - long lastAccess = tablet.getLastAccessTimestamp(); manualClock.advanceTime(Duration.ofMinutes(10)); checkNotNull(replica.getKvSnapshotManager()).triggerSnapshot(); - assertThat(tablet.getLastAccessTimestamp()).isEqualTo(lastAccess); assertThat(tablet.canRelease(Duration.ofMinutes(10).toMillis(), manualClock.milliseconds())) .isTrue(); replica.lookups(Collections.singletonList(key(1))); - assertThat(tablet.getLastAccessTimestamp()).isEqualTo(manualClock.milliseconds()); assertThat(tablet.canRelease(Duration.ofMinutes(10).toMillis(), manualClock.milliseconds())) .isFalse(); } @@ -110,7 +104,7 @@ void testFirstLookupFlushesRecoveredRecordsBeyondHighWatermark() throws Exceptio Collections.singletonList(new Object[] {1, "a"}), LogFormat.ARROW, DEFAULT_COMPRESSION)); - makeLeaderReplica(replica); + replica.makeLeader(notifyLeaderAndIsr(0)); assertThat(replica.getLogHighWatermark()).isZero(); assertThat(replica.getLocalLogEndOffset()).isEqualTo(1L); @@ -132,7 +126,8 @@ void testPutRecordsToLeaderTriggersLazyOpen() throws Exception { assertThat(kvTablet.isLazyOpen()).isFalse(); assertThat(kvTablet.getRocksDBKv()).isNull(); - LogAppendInfo appendInfo = putRecordsToLeaderAndFlush(replica, records(1, "a")); + LogAppendInfo appendInfo = + putRecordsToLeaderAndFlush(replica, genKvRecordBatch(new Object[] {1, "a"})); assertThat(appendInfo.lastOffset()).isEqualTo(0L); CommonTestUtils.waitUntil( @@ -146,54 +141,37 @@ void testPutRecordsToLeaderTriggersLazyOpen() throws Exception { .containsExactly(valueBytes(1, "a")); } - @Test - void testLookupsReopenReleasedLazyTablet() throws Exception { - Replica replica = makeLazyLeaderReplica(); - putRecordsToLeaderAndFlush(replica, records(1, "a", 2, "b")); - releaseToLazy(replica); - - List values = replica.lookups(Arrays.asList(key(1), key(2))); - - assertThat(values) - .extracting(ByteArraySlice::toByteArray) - .containsExactly(valueBytes(1, "a"), valueBytes(2, "b")); - assertThat(checkNotNull(replica.getKvTablet()).isLazyOpen()).isTrue(); - } - - @Test - void testPrefixLookupReopensReleasedLazyTablet() throws Exception { - Replica replica = makeLazyLeaderReplica(); - putRecordsToLeaderAndFlush(replica, records(1, "a", 2, "b")); - releaseToLazy(replica); - - List values = replica.prefixLookup(new byte[0]); - - assertThat(values) - .extracting(ByteArraySlice::toByteArray) - .containsExactly(valueBytes(1, "a"), valueBytes(2, "b")); - assertThat(checkNotNull(replica.getKvTablet()).isLazyOpen()).isTrue(); - } - - @Test - void testLimitScanReopensReleasedLazyTablet() throws Exception { + @ParameterizedTest + @ValueSource(strings = {"lookup", "prefix", "scan"}) + void testReadReopensReleasedTablet(String operation) throws Exception { Replica replica = makeLazyLeaderReplica(); - putRecordsToLeaderAndFlush(replica, records(1, "a", 2, "b")); + putRecordsToLeaderAndFlush( + replica, genKvRecordBatch(new Object[] {1, "a"}, new Object[] {2, "b"})); + long flushed = replica.getKvTablet().getFlushedLogOffset(); releaseToLazy(replica); - - DefaultValueRecordBatch.Builder builder = DefaultValueRecordBatch.builder(); - builder.append(DEFAULT_SCHEMA_ID, compactedRow(DATA1_ROW_TYPE, new Object[] {1, "a"})); - builder.append(DEFAULT_SCHEMA_ID, compactedRow(DATA1_ROW_TYPE, new Object[] {2, "b"})); - - DefaultValueRecordBatch values = replica.limitKvScan(2); - - assertThat(values).isEqualTo(builder.build()); - assertThat(checkNotNull(replica.getKvTablet()).isLazyOpen()).isTrue(); + assertThat(replica.getKvTablet().getFlushedLogOffset()).isEqualTo(flushed); + assertThat(replica.getRowCount()).isEqualTo(2L); + assertThat(replica.getKvTablet().isLazyOpen()).isFalse(); + if (operation.equals("scan")) { + DefaultValueRecordBatch.Builder builder = DefaultValueRecordBatch.builder(); + builder.append(DEFAULT_SCHEMA_ID, compactedRow(DATA1_ROW_TYPE, new Object[] {1, "a"})); + builder.append(DEFAULT_SCHEMA_ID, compactedRow(DATA1_ROW_TYPE, new Object[] {2, "b"})); + assertThat(replica.limitKvScan(2)).isEqualTo(builder.build()); + } else { + assertThat( + operation.equals("prefix") + ? replica.prefixLookup(new byte[0]) + : replica.lookups(Arrays.asList(key(1), key(2)))) + .extracting(ByteArraySlice::toByteArray) + .containsExactly(valueBytes(1, "a"), valueBytes(2, "b")); + } + assertThat(replica.getKvTablet().isLazyOpen()).isTrue(); } @Test void testScannerPreventsIdleReleaseUntilClosed() throws Exception { Replica replica = makeLazyLeaderReplica(); - putRecordsToLeaderAndFlush(replica, records(1, "a")); + putRecordsToLeaderAndFlush(replica, genKvRecordBatch(new Object[] {1, "a"})); releaseToLazy(replica); ScannerContext scanner = @@ -214,167 +192,80 @@ void testScannerPreventsIdleReleaseUntilClosed() throws Exception { } @Test - void testGetRowCountReturnsCachedValueWithoutReopening() throws Exception { + void testReleasedTabletUsesCommittedSnapshotSize() throws Exception { Replica replica = makeLazyLeaderReplica(); - putRecordsToLeaderAndFlush(replica, records(1, "a", 2, "b")); - releaseToLazy(replica); - - long rowCount = replica.getRowCount(); - - assertThat(rowCount).isEqualTo(2L); - assertThat(checkNotNull(replica.getKvTablet()).isLazyOpen()).isFalse(); - } - - @Test - void testLatestKvSnapshotSizeFallsBackToCommittedSnapshotWhenReleased() throws Exception { - TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID_PK, 1); - Replica replica = makeLazyLeaderReplicaWithSharedSnapshotStore(tableBucket); - putRecordsToLeaderAndFlush(replica, records(1, "a", 2, "b")); - - // Arbitrary non-zero size to verify fallback to committed snapshot - long expectedSnapshotSize = 321L; - KvSnapshotHandle snapshotHandle = - KvSnapshotHandle.create( - Collections.singletonList( - KvFileHandleAndLocalPath.of( - new KvFileHandle( - new FsPath( - tempDir.getAbsolutePath(), - "snapshot-1.sst") - .toString(), - expectedSnapshotSize), - "snapshot-1.sst")), - Collections.emptyList(), - expectedSnapshotSize); - CompletedSnapshot completedSnapshot = - new CompletedSnapshot( - tableBucket, - 1L, - new FsPath(tempDir.getAbsolutePath(), "snapshot-1"), - snapshotHandle, - replica.getLocalLogEndOffset(), - 2L, - null); - snapshotReporter.commitKvSnapshot(completedSnapshot, 0, 0); - + putRecordsToLeaderAndFlush( + replica, genKvRecordBatch(new Object[] {1, "a"}, new Object[] {2, "b"})); + checkNotNull(replica.getKvSnapshotManager()).triggerSnapshot(); + CompletedSnapshot snapshot = + snapshotReporter.waitUntilSnapshotComplete(replica.getTableBucket(), 0); + assertThat(snapshot.getSnapshotSize()).isPositive(); releaseToLazy(replica); - assertThat(replica.getKvSnapshotManager()).isNull(); - assertThat(replica.getLatestKvSnapshotSize()) - .isEqualTo(completedSnapshot.getSnapshotSize()); - assertThat(replica.logicalStorageKvSize()).isEqualTo(completedSnapshot.getSnapshotSize()); + assertThat(replica.getLatestKvSnapshotSize()).isEqualTo(snapshot.getSnapshotSize()); + assertThat(replica.logicalStorageKvSize()).isEqualTo(snapshot.getSnapshotSize()); } - @Test - void testMakeFollowerDropsLazyTablet() throws Exception { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testBecomingFollowerDropsTablet(boolean opened) throws Exception { Replica replica = makeLazyLeaderReplica(); - KvTablet kvTablet = checkNotNull(replica.getKvTablet()); - assertThat(kvTablet.isLazyOpen()).isFalse(); - assertThat(kvTablet.getRocksDBKv()).isNull(); - - replica.makeFollower( - notifyLeaderAndIsr( - new TableBucket(DATA1_TABLE_ID_PK, 1), - Collections.singletonList(TABLET_SERVER_ID), - Collections.singletonList(TABLET_SERVER_ID), - 1)); - - assertThat(replica.getKvTablet()).isNull(); - } - - @Test - void testMakeFollowerDropsOpenLazyTablet() throws Exception { - Replica replica = makeLazyLeaderReplica(); - putRecordsToLeaderAndFlush(replica, records(1, "a")); - KvTablet kvTablet = checkNotNull(replica.getKvTablet()); - assertThat(kvTablet.isLazyOpen()).isTrue(); - - replica.makeFollower( - notifyLeaderAndIsr( - new TableBucket(DATA1_TABLE_ID_PK, 1), - Collections.singletonList(TABLET_SERVER_ID), - Collections.singletonList(TABLET_SERVER_ID), - 1)); - + if (opened) { + putRecordsToLeaderAndFlush(replica, genKvRecordBatch(new Object[] {1, "a"})); + } + replica.makeFollower(notifyLeaderAndIsr(1)); assertThat(replica.getKvTablet()).isNull(); + assertThat(kvManager.getKv(replica.getTableBucket())).isEmpty(); } @Test void testReopenFallsBackToFullInitWhenLocalDataCorrupted() throws Exception { Replica replica = makeLazyLeaderReplica(); - putRecordsToLeaderAndFlush(replica, records(1, "a", 2, "b")); + putRecordsToLeaderAndFlush( + replica, genKvRecordBatch(new Object[] {1, "a"}, new Object[] {2, "b"})); - // Get the tablet directory path before release - KvTablet kvTablet = checkNotNull(replica.getKvTablet()); - File tabletDir = - kvManager.getTabletDirPath( - replica.getLogTablet().getDataDir(), - DATA1_PHYSICAL_TABLE_PATH_PK, - new TableBucket(DATA1_TABLE_ID_PK, 1)); + File tabletDir = replica.getKvTablet().getKvTabletDir(); releaseToLazy(replica); - // Corrupt the local directory by deleting it assertThat(tabletDir).exists(); deleteDirectory(tabletDir); assertThat(tabletDir).doesNotExist(); - // Trigger reopen — should fall back to full init (no snapshot = fresh tablet) List values = replica.lookups(Collections.singletonList(key(1))); - // Tablet should be open and functional after fallback assertThat(checkNotNull(replica.getKvTablet()).isLazyOpen()).isTrue(); assertThat(values.get(0).toByteArray()).isEqualTo(valueBytes(1, "a")); - // Verify the tablet is fully functional — can write new data - putRecordsToLeaderAndFlush(replica, records(3, "c")); + putRecordsToLeaderAndFlush(replica, genKvRecordBatch(new Object[] {3, "c"})); assertThat(replica.lookups(Collections.singletonList(key(3)))) .extracting(ByteArraySlice::toByteArray) .containsExactly(valueBytes(3, "c")); } private Replica makeLazyLeaderReplica() throws Exception { - Replica replica = - makeKvReplica(DATA1_PHYSICAL_TABLE_PATH_PK, new TableBucket(DATA1_TABLE_ID_PK, 1)); - makeLeaderReplica(replica); - return replica; - } - - private Replica makeLazyLeaderReplicaWithSharedSnapshotStore(TableBucket tableBucket) - throws Exception { Replica replica = makeKvReplica( DATA1_PHYSICAL_TABLE_PATH_PK, - tableBucket, + new TableBucket(DATA1_TABLE_ID_PK, 1), new TestSnapshotContext( conf.getString(ConfigOptions.REMOTE_DATA_DIR), snapshotReporter)); - makeLeaderReplica(replica); + replica.makeLeader(notifyLeaderAndIsr(0)); return replica; } - private void makeLeaderReplica(Replica replica) throws Exception { - replica.makeLeader( - notifyLeaderAndIsr( - new TableBucket(DATA1_TABLE_ID_PK, 1), - Collections.singletonList(TABLET_SERVER_ID), - Collections.singletonList(TABLET_SERVER_ID), - 0)); - } - - private NotifyLeaderAndIsrData notifyLeaderAndIsr( - TableBucket tableBucket, List replicas, List isr, int leaderEpoch) { + private NotifyLeaderAndIsrData notifyLeaderAndIsr(int epoch) { return new NotifyLeaderAndIsrData( DATA1_PHYSICAL_TABLE_PATH_PK, - tableBucket, - replicas, + new TableBucket(DATA1_TABLE_ID_PK, 1), + Collections.singletonList(TABLET_SERVER_ID), new LeaderAndIsr( TABLET_SERVER_ID, - leaderEpoch, - isr, + epoch, + Collections.singletonList(TABLET_SERVER_ID), Collections.emptyList(), - org.apache.fluss.server.coordinator.CoordinatorContext - .INITIAL_COORDINATOR_EPOCH, - leaderEpoch)); + 0, + epoch)); } private LogAppendInfo putRecordsToLeaderAndFlush(Replica replica, KvRecordBatch kvRecords) @@ -400,21 +291,6 @@ private void releaseToLazy(Replica replica) throws Exception { assertThat(kvTablet.getRocksDBKv()).isNull(); } - private KvRecordBatch records(Object... values) throws Exception { - KvRecordTestUtils.KvRecordFactory recordFactory = - KvRecordTestUtils.KvRecordFactory.of(DATA1_ROW_TYPE); - KvRecordTestUtils.KvRecordBatchFactory batchFactory = - KvRecordTestUtils.KvRecordBatchFactory.of(DEFAULT_SCHEMA_ID); - - KvRecord[] records = new KvRecord[values.length / 2]; - for (int i = 0; i < values.length; i += 2) { - records[i / 2] = - recordFactory.ofRecord( - key((Integer) values[i]), new Object[] {values[i], values[i + 1]}); - } - return batchFactory.ofRecords(records); - } - private byte[] key(int id) { return new CompactedKeyEncoder(DATA1_ROW_TYPE, new int[] {0}) .encodeKey(compactedRow(DATA1_ROW_TYPE, new Object[] {id, ""})); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java index 1956c2bbab7..fb25d48f482 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java @@ -1280,39 +1280,32 @@ private void verifyGetKeyValues( @Test void testGetRowCountPkTableWithNullKvTablet() throws Exception { - // Create a PK table replica but do NOT make it leader, so kvTablet stays null. + TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID_PK, 1); Replica kvReplica = makeKvReplica(DATA1_PHYSICAL_TABLE_PATH_PK, tableBucket); - // Verify preconditions: this is a PK table and kvTablet is null. assertThat(kvReplica.isKvTable()).isTrue(); assertThat(kvReplica.getKvTablet()).isNull(); - // Append some log records so logTablet has a non-zero row count. LogTablet logTablet = kvReplica.getLogTablet(); MemoryLogRecords records = genMemoryLogRecordsByObject(DATA1); logTablet.appendAsLeader(records); logTablet.updateHighWatermark(logTablet.localLogEndOffset()); assertThat(logTablet.getRowCount()).isGreaterThan(0); - // getRowCount() should return 0 for a PK table when kvTablet is null, - // NOT the logTablet row count. assertThat(kvReplica.getRowCount()).isEqualTo(0L); } @Test void testCreateKvRollbackOnAllRetriesFailed() throws Exception { - // Create a SnapshotContext whose getSnapshotDataDownloader always throws. - // This forces initKvTablet() to fail on every retry attempt because the - // snapshot download will always fail. + TableBucket tableBucket = new TableBucket(DATA1_TABLE_ID_PK, 1); TestSnapshotContext failingSnapshotContext = new TestSnapshotContext(conf.getString(ConfigOptions.REMOTE_DATA_DIR)) { @Override public FunctionWithException getLatestCompletedSnapshotProvider() { - // Return a provider that always returns a fake snapshot, - // so initKvTablet takes the snapshot-restore path + return tb -> new CompletedSnapshot( tb, @@ -1329,17 +1322,8 @@ void testCreateKvRollbackOnAllRetriesFailed() throws Exception { @Override public KvSnapshotDataDownloader getSnapshotDataDownloader() { - // Return a downloader that always throws to simulate download failure - return new KvSnapshotDataDownloader( - java.util.concurrent.Executors.newSingleThreadExecutor()) { - @Override - public void transferAllDataToDirectory( - KvSnapshotDownloadSpec kvSnapshotDownloadSpec, - org.apache.fluss.utils.CloseableRegistry closeableRegistry) - throws Exception { - throw new IOException("Simulated snapshot download failure"); - } - }; + + throw new IllegalStateException("Snapshot download unavailable"); } }; @@ -1349,10 +1333,8 @@ public void transferAllDataToDirectory( assertThatThrownBy(() -> makeKvReplicaAsLeader(kvReplica)) .isInstanceOf(org.apache.fluss.exception.KvStorageException.class); - // kvTablet should be null since all retries failed assertThat(kvReplica.getKvTablet()).isNull(); - // KvManager should not have a half-initialized entry for this bucket assertThat(kvManager.getKv(tableBucket)).isEmpty(); } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java index 12933e36fbe..07e87adec67 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManagerTest.java @@ -1433,7 +1433,10 @@ private static void assertHistoricalValue( TableInfo tableInfo, InternalRow expectedRow) throws Exception { - KvStateLookupResult result = kvTablet.lookupHistoricalLocal(originalPartition, primaryKey); + KvStateLookupResult result; + try (KvTablet.Guard guard = kvTablet.acquireGuard()) { + result = guard.getTablet().lookupHistoricalLocal(originalPartition, primaryKey); + } assertThat(result.isPresent()).isTrue(); BinaryValue value = new ValueDecoder(