From a433253e0eb2620c940b9283ffe0836a6a2cb2c1 Mon Sep 17 00:00:00 2001 From: zhangjunfan Date: Wed, 26 Aug 2026 16:51:42 +0800 Subject: [PATCH 1/8] [server] Add support of sharable file cache capacity for historical partition lookup --- .../apache/fluss/config/ConfigOptions.java | 2 +- .../fluss/lake/lakestorage/LakeStorage.java | 73 +--- .../lakestorage/LakeTableLookupRuntime.java | 90 +++++ .../lakestorage/PluginLakeStorageWrapper.java | 45 ++- .../lake/lakestorage/LakeStorageTest.java | 38 ++- .../fluss/lake/paimon/PaimonLakeStorage.java | 68 +++- .../lookup/NamespacedLookupFileCache.java | 205 ++++++++++++ .../lookup/PaimonLakeTableLookuper.java | 66 ++-- .../paimon/lookup/PaimonLocalTableQuery.java | 316 ++++++++++++++++++ .../paimon/lookup/SharedLookupFileCache.java | 112 +++++++ .../lookup/PaimonLakeTableLookuperTest.java | 128 ++++++- .../lookup/SharedLookupFileCacheTest.java | 93 ++++++ .../HistoricalLakeLookupManager.java | 148 ++++---- .../lakehouse/TestingPaimonStoragePlugin.java | 20 ++ .../HistoricalLakeLookupManagerTest.java | 44 ++- 15 files changed, 1233 insertions(+), 215 deletions(-) create mode 100644 fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookupRuntime.java create mode 100644 fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/NamespacedLookupFileCache.java create mode 100644 fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLocalTableQuery.java create mode 100644 fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/SharedLookupFileCache.java create mode 100644 fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/SharedLookupFileCacheTest.java diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 370e5f8e7a9..245ec95a486 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -429,7 +429,7 @@ public class ConfigOptions { .defaultValue(0.10) .withDescription( "The maximum fraction of the total capacity of the volume containing the first available data directory allocated to historical partition lookup caches on a TabletServer. " - + "Up to ten table lookupers are cached, and each receives one tenth of this capacity. Historical lookup cache files are stored under that data directory; additional data volumes are not used. " + + "Up to ten table lookupers share this capacity. Historical lookup cache files are stored under that data directory; additional data volumes are not used. " + "The valid range is (0.0, 1.0]."); public static final ConfigOption diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java index 478e8d4a3ec..bba509188f6 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java @@ -18,15 +18,10 @@ package org.apache.fluss.lake.lakestorage; import org.apache.fluss.annotation.PublicEvolving; -import org.apache.fluss.config.TableConfig; import org.apache.fluss.lake.source.LakeSource; import org.apache.fluss.lake.writer.LakeTieringFactory; -import org.apache.fluss.metadata.LakeLookupMode; import org.apache.fluss.metadata.TablePath; -import static org.apache.fluss.utils.Preconditions.checkArgument; -import static org.apache.fluss.utils.Preconditions.checkNotNull; - /** * The LakeStorage interface defines how to implement lakehouse storage system such as Paimon and * Iceberg. It provides a method to create a lake tiering factory. @@ -57,71 +52,15 @@ public interface LakeStorage { LakeSource createLakeSource(TablePath tablePath); /** - * Creates a table-level point lookuper for the specified lake table. + * Creates a TabletServer-scoped runtime for lake table point lookup. * - * @param tablePath the logical path identifying the table in the lakehouse storage - * @param context runtime context for creating the lookuper - * @return a table-level point lookuper + * @param ioTmpDir local directory shared by lookupers for temporary files + * @param lookupCacheMaxDiskBytes maximum local lookup cache size in bytes + * @return the lookup runtime */ - default LakeTableLookuper createLakeTableLookuper( - TablePath tablePath, LookuperContext context) { + default LakeTableLookupRuntime createLakeTableLookupRuntime( + String ioTmpDir, long lookupCacheMaxDiskBytes) { throw new UnsupportedOperationException( "Point lookup is not supported for this lake storage."); } - - /** Runtime context for creating a lake table lookuper. */ - final class LookuperContext { - private final String ioTmpDir; - private final TableConfig tableConfig; - private final long lookupCacheMaxDiskBytes; - private final Runnable diskWriteGuard; - private final LakeLookupMode lookupMode; - - /** - * Creates a lookuper context. - * - * @param ioTmpDir local directory for temporary files used by the lookuper - * @param tableConfig configuration of the Fluss table - * @param lookupCacheMaxDiskBytes maximum local lookup cache size in bytes - * @param diskWriteGuard guard invoked before creating a local lookup cache file - */ - public LookuperContext( - String ioTmpDir, - TableConfig tableConfig, - long lookupCacheMaxDiskBytes, - Runnable diskWriteGuard) { - this.ioTmpDir = checkNotNull(ioTmpDir, "ioTmpDir must not be null."); - this.tableConfig = checkNotNull(tableConfig, "tableConfig must not be null."); - checkArgument( - lookupCacheMaxDiskBytes > 0, "lookupCacheMaxDiskBytes must be greater than 0."); - this.lookupCacheMaxDiskBytes = lookupCacheMaxDiskBytes; - this.diskWriteGuard = checkNotNull(diskWriteGuard, "diskWriteGuard must not be null."); - this.lookupMode = tableConfig.getHistoricalLookupMode(); - } - - /** Returns the local directory for temporary files used by the lookuper. */ - public String ioTmpDir() { - return ioTmpDir; - } - - /** Returns the configuration of the Fluss table. */ - public TableConfig tableConfig() { - return tableConfig; - } - - /** Returns the maximum local lookup cache size in bytes. */ - public long lookupCacheMaxDiskBytes() { - return lookupCacheMaxDiskBytes; - } - - /** Returns the guard invoked before creating a local lookup cache file. */ - public Runnable diskWriteGuard() { - return diskWriteGuard; - } - - /** Returns the mode used to look up historical data. */ - public LakeLookupMode lookupMode() { - return lookupMode; - } - } } diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookupRuntime.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookupRuntime.java new file mode 100644 index 00000000000..2eb41b2ee26 --- /dev/null +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookupRuntime.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.lake.lakestorage; + +import org.apache.fluss.annotation.PublicEvolving; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.TableConfig; +import org.apache.fluss.metadata.TablePath; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** TabletServer-scoped runtime for creating lake table lookupers. */ +@PublicEvolving +public interface LakeTableLookupRuntime extends AutoCloseable { + + /** + * Creates a table-level point lookuper for the specified lake table. + * + * @param tablePath the logical path identifying the table in the lakehouse storage + * @param context runtime context for creating the lookuper + * @return a table-level point lookuper + */ + LakeTableLookuper createLakeTableLookuper(TablePath tablePath, Context context); + + /** Updates the maximum local lookup cache size in bytes. */ + void updateLookupCacheMaxDiskBytes(long lookupCacheMaxDiskBytes); + + /** Runtime context for creating a lake table lookuper. */ + final class Context { + private final Configuration lakeConfiguration; + private final String cacheNamespace; + private final TableConfig tableConfig; + private final Runnable diskWriteGuard; + + /** + * Creates a lookuper context. + * + * @param lakeConfiguration configuration of the lake storage for this lookuper + * @param cacheNamespace namespace identifying cache entries owned by this lookuper + * @param tableConfig configuration of the Fluss table + * @param diskWriteGuard guard invoked before creating a local lookup cache file + */ + public Context( + Configuration lakeConfiguration, + String cacheNamespace, + TableConfig tableConfig, + Runnable diskWriteGuard) { + this.lakeConfiguration = + checkNotNull(lakeConfiguration, "lakeConfiguration must not be null."); + this.cacheNamespace = checkNotNull(cacheNamespace, "cacheNamespace must not be null."); + this.tableConfig = checkNotNull(tableConfig, "tableConfig must not be null."); + this.diskWriteGuard = checkNotNull(diskWriteGuard, "diskWriteGuard must not be null."); + } + + /** Returns the lake storage configuration for this lookuper. */ + public Configuration lakeConfiguration() { + return lakeConfiguration; + } + + /** Returns the namespace identifying cache entries owned by this lookuper. */ + public String cacheNamespace() { + return cacheNamespace; + } + + /** Returns the configuration of the Fluss table. */ + public TableConfig tableConfig() { + return tableConfig; + } + + /** Returns the guard invoked before creating a local lookup cache file. */ + public Runnable diskWriteGuard() { + return diskWriteGuard; + } + } +} diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java index 8c73e1b186b..2a81bdda7d9 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java @@ -139,13 +139,54 @@ public LakeSource createLakeSource(TablePath tablePath) { } @Override - public LakeTableLookuper createLakeTableLookuper( - TablePath tablePath, LookuperContext context) { + public LakeTableLookupRuntime createLakeTableLookupRuntime( + String ioTmpDir, long lookupCacheMaxDiskBytes) { + try (TemporaryClassLoaderContext ignored = TemporaryClassLoaderContext.of(loader)) { + return new ClassLoaderFixingLakeTableLookupRuntime( + inner.createLakeTableLookupRuntime(ioTmpDir, lookupCacheMaxDiskBytes), + loader); + } + } + } + + static class ClassLoaderFixingLakeTableLookupRuntime + implements LakeTableLookupRuntime, WrappingProxy { + + private final LakeTableLookupRuntime inner; + private final ClassLoader loader; + + private ClassLoaderFixingLakeTableLookupRuntime( + LakeTableLookupRuntime inner, ClassLoader loader) { + this.inner = inner; + this.loader = loader; + } + + @Override + public LakeTableLookuper createLakeTableLookuper(TablePath tablePath, Context context) { try (TemporaryClassLoaderContext ignored = TemporaryClassLoaderContext.of(loader)) { return new ClassLoaderFixingLakeTableLookuper( inner.createLakeTableLookuper(tablePath, context), loader); } } + + @Override + public void updateLookupCacheMaxDiskBytes(long lookupCacheMaxDiskBytes) { + try (TemporaryClassLoaderContext ignored = TemporaryClassLoaderContext.of(loader)) { + inner.updateLookupCacheMaxDiskBytes(lookupCacheMaxDiskBytes); + } + } + + @Override + public void close() throws Exception { + try (TemporaryClassLoaderContext ignored = TemporaryClassLoaderContext.of(loader)) { + inner.close(); + } + } + + @Override + public LakeTableLookupRuntime getWrappedDelegate() { + return inner; + } } static class ClassLoaderFixingLakeTableLookuper diff --git a/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java b/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java index b527bb6e483..08f0d785c18 100644 --- a/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java @@ -96,6 +96,19 @@ void testWithPluginManager() throws Exception { ((PluginLakeStorageWrapper.ClassLoaderFixingLakeCatalog) lakeCatalog) .getWrappedDelegate()) .isInstanceOf(TestPaimonLakeCatalog.class); + + LakeTableLookupRuntime lookupRuntime = + lakeStorage.createLakeTableLookupRuntime("lookup-dir", 1024L); + assertThat(lookupRuntime) + .isInstanceOf( + PluginLakeStorageWrapper.ClassLoaderFixingLakeTableLookupRuntime.class); + TestLakeTableLookupRuntime innerLookupRuntime = + (TestLakeTableLookupRuntime) + ((PluginLakeStorageWrapper.ClassLoaderFixingLakeTableLookupRuntime) + lookupRuntime) + .getWrappedDelegate(); + lookupRuntime.close(); + assertThat(innerLookupRuntime.closed).isTrue(); } private static class TestingPluginManager implements PluginManager { @@ -129,7 +142,6 @@ public LakeStorage createLakeStorage(Configuration configuration) { } private static class TestPaimonLakeStorage implements LakeStorage { - public TestPaimonLakeStorage() {} @Override @@ -146,6 +158,30 @@ public TestPaimonLakeCatalog createLakeCatalog() { public LakeSource createLakeSource(TablePath tablePath) { throw new UnsupportedOperationException("Not implemented"); } + + @Override + public LakeTableLookupRuntime createLakeTableLookupRuntime( + String ioTmpDir, long lookupCacheMaxDiskBytes) { + return new TestLakeTableLookupRuntime(); + } + } + + private static class TestLakeTableLookupRuntime implements LakeTableLookupRuntime { + + private boolean closed; + + @Override + public LakeTableLookuper createLakeTableLookuper(TablePath tablePath, Context context) { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public void updateLookupCacheMaxDiskBytes(long lookupCacheMaxDiskBytes) {} + + @Override + public void close() { + closed = true; + } } private static class TestPaimonLakeCatalog implements LakeCatalog { diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java index cace6129a87..5779444a920 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java @@ -19,9 +19,11 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.lake.lakestorage.LakeStorage; +import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.lake.paimon.lookup.PaimonLakeTableLookuper; import org.apache.fluss.lake.paimon.lookup.PaimonScanBasedTableLookuper; +import org.apache.fluss.lake.paimon.lookup.SharedLookupFileCache; import org.apache.fluss.lake.paimon.source.PaimonLakeSource; import org.apache.fluss.lake.paimon.source.PaimonSplit; import org.apache.fluss.lake.paimon.tiering.PaimonCommittable; @@ -31,6 +33,14 @@ import org.apache.fluss.lake.writer.LakeTieringFactory; import org.apache.fluss.metadata.LakeLookupMode; import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.utils.IOUtils; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.disk.IOManager; +import org.apache.paimon.options.MemorySize; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; /** Paimon implementation of {@link LakeStorage}. */ public class PaimonLakeStorage implements LakeStorage { @@ -57,16 +67,54 @@ public LakeSource createLakeSource(TablePath tablePath) { } @Override - public LakeTableLookuper createLakeTableLookuper(TablePath tablePath, LookuperContext context) { - if (context.lookupMode() == LakeLookupMode.SCAN) { - return new PaimonScanBasedTableLookuper(paimonConfig, tablePath, context.tableConfig()); + public LakeTableLookupRuntime createLakeTableLookupRuntime( + String ioTmpDir, long lookupCacheMaxDiskBytes) { + return new PaimonLakeTableLookupRuntime(ioTmpDir, lookupCacheMaxDiskBytes); + } + + /** Paimon lookup runtime sharing one I/O manager across table lookupers. */ + private static final class PaimonLakeTableLookupRuntime implements LakeTableLookupRuntime { + private final IOManager ioManager; + private final SharedLookupFileCache lookupFileCache; + + private PaimonLakeTableLookupRuntime(String ioTmpDir, long lookupCacheMaxDiskBytes) { + checkArgument( + lookupCacheMaxDiskBytes > 0, "lookupCacheMaxDiskBytes must be greater than 0."); + this.ioManager = IOManager.create(checkNotNull(ioTmpDir, "ioTmpDir must not be null.")); + // ponytail: one runtime-wide retention; add a server option if this needs tuning. + this.lookupFileCache = + new SharedLookupFileCache( + CoreOptions.LOOKUP_CACHE_FILE_RETENTION.defaultValue(), + new MemorySize(lookupCacheMaxDiskBytes)); + } + + @Override + public LakeTableLookuper createLakeTableLookuper(TablePath tablePath, Context context) { + if (context.tableConfig().getHistoricalLookupMode() == LakeLookupMode.SCAN) { + return new PaimonScanBasedTableLookuper( + new Configuration(context.lakeConfiguration()), tablePath, context.tableConfig()); + } + return new PaimonLakeTableLookuper( + new Configuration(context.lakeConfiguration()), + tablePath, + ioManager, + lookupFileCache, + context.cacheNamespace(), + context.tableConfig(), + context.diskWriteGuard()); + } + + @Override + public void updateLookupCacheMaxDiskBytes(long lookupCacheMaxDiskBytes) { + checkArgument( + lookupCacheMaxDiskBytes > 0, "lookupCacheMaxDiskBytes must be greater than 0."); + lookupFileCache.updateMaxDiskSize(new MemorySize(lookupCacheMaxDiskBytes)); + } + + @Override + public void close() { + IOUtils.closeQuietly(lookupFileCache, "shared Paimon lookup-file cache"); + IOUtils.closeQuietly(ioManager, "shared Paimon lookup IO manager"); } - return new PaimonLakeTableLookuper( - paimonConfig, - tablePath, - context.ioTmpDir(), - context.tableConfig(), - context.lookupCacheMaxDiskBytes(), - context.diskWriteGuard()); } } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/NamespacedLookupFileCache.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/NamespacedLookupFileCache.java new file mode 100644 index 00000000000..0eb408101ff --- /dev/null +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/NamespacedLookupFileCache.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.lake.paimon.lookup; + +import org.apache.paimon.mergetree.LookupFile; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Policy; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.stats.CacheStats; + +import javax.annotation.Nullable; + +import java.util.AbstractMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Function; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Namespace-scoped cache view delegating all storage to a shared lookup-file cache. */ +final class NamespacedLookupFileCache implements Cache { + + private final Cache sharedCache; + private final String namespace; + + NamespacedLookupFileCache( + Cache sharedCache, String namespace) { + this.sharedCache = checkNotNull(sharedCache, "sharedCache must not be null."); + this.namespace = checkNotNull(namespace, "namespace must not be null."); + } + + @Override + public @Nullable LookupFile getIfPresent(Object fileName) { + return fileName instanceof String ? sharedCache.getIfPresent(key((String) fileName)) : null; + } + + @Override + public LookupFile get( + String fileName, Function mappingFunction) { + return sharedCache.get(key(fileName), ignored -> mappingFunction.apply(fileName)); + } + + @Override + public Map getAllPresent(Iterable fileNames) { + Map result = new LinkedHashMap<>(); + for (Object fileName : fileNames) { + LookupFile lookupFile = getIfPresent(fileName); + if (lookupFile != null) { + result.put((String) fileName, lookupFile); + } + } + return result; + } + + @Override + public void put(String fileName, LookupFile lookupFile) { + sharedCache.put(key(fileName), lookupFile); + } + + @Override + public void putAll(Map entries) { + entries.forEach(this::put); + } + + @Override + public void invalidate(Object fileName) { + if (fileName instanceof String) { + sharedCache.invalidate(key((String) fileName)); + } + } + + @Override + public void invalidateAll(Iterable fileNames) { + for (Object fileName : fileNames) { + invalidate(fileName); + } + } + + @Override + public void invalidateAll() { + // ponytail: O(n) namespace scan; add a namespace index if cache cardinality makes close + // slow. + Set keys = new HashSet<>(); + for (SharedLookupFileCache.Key key : sharedCache.asMap().keySet()) { + if (key.namespace.equals(namespace)) { + keys.add(key); + } + } + sharedCache.invalidateAll(keys); + } + + @Override + public long estimatedSize() { + return sharedCache.asMap().keySet().stream() + .filter(key -> key.namespace.equals(namespace)) + .count(); + } + + @Override + public CacheStats stats() { + return sharedCache.stats(); + } + + @Override + public ConcurrentMap asMap() { + return new NamespacedMap(); + } + + @Override + public void cleanUp() { + sharedCache.cleanUp(); + } + + @Override + public Policy policy() { + throw new UnsupportedOperationException( + "Policy access is not supported by the namespaced cache view."); + } + + /** Concurrent-map view required by Paimon's closed-entry removal path. */ + private final class NamespacedMap extends AbstractMap + implements ConcurrentMap { + + @Override + public Set> entrySet() { + Set> entries = new HashSet<>(); + sharedCache + .asMap() + .forEach( + (key, lookupFile) -> { + if (key.namespace.equals(namespace)) { + entries.add( + new SimpleImmutableEntry<>(key.fileName, lookupFile)); + } + }); + return entries; + } + + @Override + public @Nullable LookupFile get(Object fileName) { + return fileName instanceof String + ? sharedCache.asMap().get(key((String) fileName)) + : null; + } + + @Override + public @Nullable LookupFile put(String fileName, LookupFile lookupFile) { + return sharedCache.asMap().put(key(fileName), lookupFile); + } + + @Override + public @Nullable LookupFile remove(Object fileName) { + return fileName instanceof String + ? sharedCache.asMap().remove(key((String) fileName)) + : null; + } + + @Override + public boolean remove(Object fileName, Object lookupFile) { + return fileName instanceof String + && sharedCache.asMap().remove(key((String) fileName), lookupFile); + } + + @Override + public @Nullable LookupFile putIfAbsent(String fileName, LookupFile lookupFile) { + return sharedCache.asMap().putIfAbsent(key(fileName), lookupFile); + } + + @Override + public boolean replace(String fileName, LookupFile oldValue, LookupFile newValue) { + return sharedCache.asMap().replace(key(fileName), oldValue, newValue); + } + + @Override + public @Nullable LookupFile replace(String fileName, LookupFile lookupFile) { + return sharedCache.asMap().replace(key(fileName), lookupFile); + } + + @Override + public void clear() { + invalidateAll(); + } + } + + private SharedLookupFileCache.Key key(String fileName) { + return new SharedLookupFileCache.Key(namespace, fileName); + } +} diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java index 2d2ecac1903..34e0103d159 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java @@ -19,7 +19,6 @@ import org.apache.fluss.bucketing.BucketingFunction; import org.apache.fluss.config.Configuration; -import org.apache.fluss.config.MemorySize; import org.apache.fluss.config.TableConfig; import org.apache.fluss.exception.DiskWriteLockedException; import org.apache.fluss.exception.KvStorageException; @@ -35,7 +34,6 @@ import org.apache.fluss.utils.ExceptionUtils; import org.apache.fluss.utils.IOUtils; -import org.apache.paimon.CoreOptions; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.CatalogFactory; @@ -47,7 +45,6 @@ import org.apache.paimon.memory.MemorySegment; import org.apache.paimon.options.Options; import org.apache.paimon.table.FileStoreTable; -import org.apache.paimon.table.query.LocalTableQuery; import org.apache.paimon.table.sink.RowPartitionKeyExtractor; import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.table.source.InnerTableScan; @@ -74,15 +71,15 @@ import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toFlussValue; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimon; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimonPartition; -import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; /** * Paimon implementation of {@link LakeTableLookuper} for primary-key tables. * - *

The catalog, table, local query, and I/O manager are initialized lazily on the first lookup. - * For each partition and bucket, the lookuper scans the latest Paimon snapshot once and registers - * its data files with {@link LocalTableQuery}. Paimon then creates local lookup files lazily as + *

The catalog, table, and local query are initialized lazily on the first lookup. The I/O + * manager is supplied by the lake storage and may be shared with other table lookupers. For each + * partition and bucket, the lookuper scans the latest Paimon snapshot once and registers its data + * files with {@link PaimonLocalTableQuery}. Paimon then creates local lookup files lazily as * individual remote data files are queried. * *

A cached partition-bucket file set can become stale when Paimon compaction replaces its data @@ -95,7 +92,7 @@ * registered partition-bucket and updates its file set in place. Paimon keeps lookup files for data * files that remain active and lazily downloads lookup files only for newly added data files. * - *

Calls to {@link LocalTableQuery#lookup} are serialized because Paimon 2.0 shares mutable + *

Calls to {@link PaimonLocalTableQuery#lookup} are serialized because Paimon 2.0 shares mutable * lookup-store comparator state across local lookup files. * *

Close is expected only after the owner has drained active lookups. It is synchronized with @@ -110,9 +107,10 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper { private final Configuration paimonConfig; private final TablePath tablePath; - private final String ioTmpDir; + private final IOManager ioManager; + private final SharedLookupFileCache lookupFileCache; + private final String cacheNamespace; private final TableConfig tableConfig; - private final long lookupCacheMaxDiskBytes; private final Runnable diskWriteGuard; private final ThreadLocal lookupFileDownloaded; @@ -128,36 +126,36 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper { private @Nullable Catalog catalog; private @Nullable FileStoreTable fileStoreTable; - private @Nullable IOManager ioManager; private @Nullable List trimmedPrimaryKeys; // CompactedKeyDecoder contains immutable type metadata and creates all decode state per // invocation, so it can be shared by concurrent lookups. private @Nullable CompactedKeyDecoder compactedKeyDecoder; - private volatile @Nullable LocalTableQuery localTableQuery; + private volatile @Nullable PaimonLocalTableQuery localTableQuery; // Guarded by lookupStateLock. private volatile boolean closed; - /** Creates a lookuper with the specified local lookup cache limit. */ + /** Creates a lookuper using an I/O manager shared with other table lookupers. */ public PaimonLakeTableLookuper( Configuration paimonConfig, TablePath tablePath, - String ioTmpDir, + IOManager ioManager, + SharedLookupFileCache lookupFileCache, + String cacheNamespace, TableConfig tableConfig, - long lookupCacheMaxDiskBytes, Runnable diskWriteGuard) { this.paimonConfig = checkNotNull(paimonConfig, "paimonConfig must not be null."); this.tablePath = checkNotNull(tablePath, "tablePath must not be null."); - this.ioTmpDir = checkNotNull(ioTmpDir, "ioTmpDir must not be null."); + this.lookupFileCache = checkNotNull(lookupFileCache, "lookupFileCache must not be null."); + this.cacheNamespace = checkNotNull(cacheNamespace, "cacheNamespace must not be null."); this.tableConfig = checkNotNull(tableConfig, "tableConfig must not be null."); - checkArgument( - lookupCacheMaxDiskBytes > 0, "lookupCacheMaxDiskBytes must be greater than 0."); - this.lookupCacheMaxDiskBytes = lookupCacheMaxDiskBytes; this.diskWriteGuard = checkNotNull(diskWriteGuard, "diskWriteGuard must not be null."); this.lookupFileDownloaded = new ThreadLocal<>(); this.paimonLookupLock = new Object(); this.lookupStateLock = new Object(); + this.ioManager = + new TrackingIOManager(checkNotNull(ioManager, "ioManager must not be null.")); this.registeredFiles = new ConcurrentHashMap<>(); this.pendingRefreshRequests = new AtomicLong(); this.totalBucketsByPartition = new ConcurrentHashMap<>(); @@ -196,14 +194,12 @@ public void close() { } closed = true; IOUtils.closeQuietly(localTableQuery, "Paimon lookup engine"); - IOUtils.closeQuietly(ioManager, "Paimon lookup IO manager"); IOUtils.closeQuietly(catalog, "Paimon catalog"); registeredFiles.clear(); totalBucketsByPartition.clear(); localTableQuery = null; compactedKeyDecoder = null; trimmedPrimaryKeys = null; - ioManager = null; fileStoreTable = null; catalog = null; } @@ -237,16 +233,14 @@ private void initialize(RowType valueRowType) throws Exception { private void initializeLookupState(RowType valueRowType) throws Exception { Catalog newCatalog = null; - IOManager newIOManager = null; - LocalTableQuery newLocalTableQuery = null; + PaimonLocalTableQuery newLocalTableQuery = null; boolean initialized = false; try { newCatalog = CatalogFactory.createCatalog( CatalogContext.create(Options.fromMap(paimonConfig.toMap()))); FileStoreTable newFileStoreTable = - withLookupCacheOptions( - (FileStoreTable) newCatalog.getTable(toPaimon(tablePath))); + (FileStoreTable) newCatalog.getTable(toPaimon(tablePath)); if (newFileStoreTable.primaryKeys().isEmpty()) { throw new UnsupportedOperationException( "Point lookup is only supported for primary-key Paimon tables."); @@ -269,16 +263,13 @@ private void initializeLookupState(RowType valueRowType) throws Exception { CompactedKeyDecoder.createKeyDecoder(valueRowType, newTrimmedPrimaryKeys); } - newIOManager = createIOManager(ioTmpDir); newLocalTableQuery = - newFileStoreTable - .newLocalTableQuery() + new PaimonLocalTableQuery(newFileStoreTable, lookupFileCache, cacheNamespace) .withValueProjection(businessFieldProjection(newFileStoreTable)) - .withIOManager(newIOManager); + .withIOManager(ioManager); catalog = newCatalog; fileStoreTable = newFileStoreTable; - ioManager = newIOManager; trimmedPrimaryKeys = newTrimmedPrimaryKeys; compactedKeyDecoder = newCompactedKeyDecoder; // Keep this volatile write last to publish all initialized fields together. @@ -287,7 +278,6 @@ private void initializeLookupState(RowType valueRowType) throws Exception { } finally { if (!initialized) { IOUtils.closeQuietly(newLocalTableQuery, "Paimon local table query"); - IOUtils.closeQuietly(newIOManager, "Paimon lookup IO manager"); IOUtils.closeQuietly(newCatalog, "Paimon catalog"); } } @@ -304,16 +294,6 @@ private void refreshFilesFromLatestSnapshot() { partitionBucket, files, () -> latestFiles.get(partitionBucket))); } - private FileStoreTable withLookupCacheOptions(FileStoreTable table) { - String key = CoreOptions.LOOKUP_CACHE_MAX_DISK_SIZE.key(); - String maxDiskSize = new MemorySize(lookupCacheMaxDiskBytes).toString(); - return table.copy(Collections.singletonMap(key, maxDiskSize)); - } - - private IOManager createIOManager(String ioTmpDir) { - return new TrackingIOManager(IOManager.create(ioTmpDir)); - } - private static int[] businessFieldProjection(FileStoreTable fileStoreTable) { List fields = fileStoreTable.schema().logicalRowType().getFields(); List projectedFields = new ArrayList<>(); @@ -665,8 +645,8 @@ public BufferFileReader createBufferFileReader(FileIOChannel.ID channelID) } @Override - public void close() throws Exception { - delegate.close(); + public void close() { + // The shared delegate is owned by PaimonLakeStorage. } } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLocalTableQuery.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLocalTableQuery.java new file mode 100644 index 00000000000..5d2ba9e262b --- /dev/null +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLocalTableQuery.java @@ -0,0 +1,316 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.lake.paimon.lookup; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.FileStore; +import org.apache.paimon.KeyValue; +import org.apache.paimon.KeyValueFileStore; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.serializer.InternalRowSerializer; +import org.apache.paimon.data.serializer.InternalSerializers; +import org.apache.paimon.data.serializer.RowCompactedSerializer; +import org.apache.paimon.deletionvectors.DeletionVector; +import org.apache.paimon.disk.IOManager; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.io.KeyValueFileReaderFactory; +import org.apache.paimon.io.cache.CacheManager; +import org.apache.paimon.lookup.LookupStoreFactory; +import org.apache.paimon.mergetree.Levels; +import org.apache.paimon.mergetree.LookupFile; +import org.apache.paimon.mergetree.LookupLevels; +import org.apache.paimon.mergetree.lookup.LookupSerializerFactory; +import org.apache.paimon.mergetree.lookup.PersistValueProcessor; +import org.apache.paimon.mergetree.lookup.RemoteLookupFileManager; +import org.apache.paimon.operation.metrics.PartialLookupMetrics; +import org.apache.paimon.options.Options; +import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.query.TableQuery; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.Filter; +import org.apache.paimon.utils.KeyComparatorSupplier; +import org.apache.paimon.utils.Preconditions; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Supplier; + +import static org.apache.paimon.lookup.LookupStoreFactory.bloomFilterBuilderFactory; +import static org.apache.paimon.mergetree.LookupFile.localFilePrefix; + +/** + * Fluss-owned fork of Paimon 2.0.0's {@code LocalTableQuery}, using a shared namespaced lookup + * cache. + */ +final class PaimonLocalTableQuery implements TableQuery { + + private final Map> tableView; + + private final CoreOptions options; + + private final Supplier> keyComparatorSupplier; + + private final KeyValueFileReaderFactory.Builder readerFactoryBuilder; + + private final LookupStoreFactory lookupStoreFactory; + + private final int startLevel; + + private IOManager ioManager; + + private final Cache lookupFileCache; + + private final RowType rowType; + private final RowType partitionType; + private final FileIO fileIO; + + @Nullable private Filter cacheRowFilter; + + @Nullable private PartialLookupMetrics partialLookupMetrics; + + PaimonLocalTableQuery( + FileStoreTable table, + SharedLookupFileCache sharedLookupFileCache, + String cacheNamespace) { + this.options = table.coreOptions(); + this.lookupFileCache = sharedLookupFileCache.namespaced(cacheNamespace); + this.tableView = new ConcurrentHashMap<>(); + FileStore tableStore = table.store(); + if (!(tableStore instanceof KeyValueFileStore)) { + throw new UnsupportedOperationException( + "Table Query only supports table with primary key."); + } + KeyValueFileStore store = (KeyValueFileStore) tableStore; + + this.readerFactoryBuilder = store.newReaderFactoryBuilder(); + this.rowType = table.schema().logicalRowType(); + this.partitionType = table.schema().logicalPartitionType(); + this.fileIO = table.fileIO(); + RowType keyType = readerFactoryBuilder.keyType(); + this.keyComparatorSupplier = new KeyComparatorSupplier(readerFactoryBuilder.keyType()); + this.lookupStoreFactory = + LookupStoreFactory.create( + options, + new CacheManager( + options.lookupCacheMaxMemory(), + options.lookupCacheHighPrioPoolRatio()), + new RowCompactedSerializer(keyType).createSliceComparator()); + startLevel = options.needLookup() ? 1 : 0; + } + + public void refreshFiles( + BinaryRow partition, + int bucket, + List beforeFiles, + List dataFiles) { + // Both tableView and its nested bucket maps are ConcurrentHashMaps; this nested + // computeIfAbsent pattern relies on each map providing atomic insertion. + BucketLookupState state = + tableView + .computeIfAbsent(partition, k -> new ConcurrentHashMap<>()) + .computeIfAbsent(bucket, k -> new BucketLookupState()); + state.lock.writeLock().lock(); + try { + if (state.lookupLevels == null) { + // Initial phase: ignore beforeFiles as they represent deletions from previous state + state.lookupLevels = createLookupLevels(partition, bucket, dataFiles); + } else { + state.lookupLevels.getLevels().update(beforeFiles, dataFiles); + } + } finally { + state.lock.writeLock().unlock(); + } + } + + private LookupLevels createLookupLevels( + BinaryRow partition, int bucket, List dataFiles) { + Levels levels = new Levels(keyComparatorSupplier.get(), dataFiles, options.numLevels()); + // TODO pass DeletionVector factory + KeyValueFileReaderFactory factory = + readerFactoryBuilder.build(partition, bucket, DeletionVector.emptyFactory()); + Options options = this.options.toConfiguration(); + + RowType readValueType = readerFactoryBuilder.readValueType(); + LookupLevels lookupLevels = + new LookupLevels<>( + schemaId -> readValueType, + 0L, + levels, + keyComparatorSupplier.get(), + readerFactoryBuilder.keyType(), + PersistValueProcessor.factory(readValueType), + LookupSerializerFactory.INSTANCE.get(), + file -> { + RecordReader reader = factory.createRecordReader(file); + if (cacheRowFilter != null) { + reader = + reader.filter( + keyValue -> cacheRowFilter.test(keyValue.value())); + } + return reader; + }, + file -> + Preconditions.checkNotNull(ioManager, "IOManager is required.") + .createChannel( + localFilePrefix( + partitionType, partition, bucket, file)) + .getPathFile(), + lookupStoreFactory, + bloomFilterBuilderFactory(options), + lookupFileCache); + + // Optimization - download lookup files if already persisted to object store + // We download these files if three conditions are met + // 1) lookup.remote-file.enabled is true - files are persisted in the first place + // 2) deletion-vectors.enabled is false - SSTables only contain row positions, not values, + // when DVs are enabled + // 3) The client is accessing the full data row, as opposed to a projection + // - The persisted remote SSTable files are created during compaction and hold the entire + // data row value + // - We could deserialize and project in memory, but we'll have to read much more data, + // not as clear of a win + boolean fullValueRead = readerFactoryBuilder.readValueType().equals(rowType); + if (this.options.lookupRemoteFileEnabled() + && !this.options.deletionVectorsEnabled() + && fullValueRead) { + // Calling the constructor tells `lookupLevels` to load remote files + new RemoteLookupFileManager<>( + fileIO, + factory.pathFactory(), + lookupLevels, + this.options.lookupRemoteLevelThreshold()); + } + + return lookupLevels; + } + + @Nullable + @Override + public InternalRow lookup(BinaryRow partition, int bucket, InternalRow key) throws IOException { + PartialLookupMetrics currentMetrics = partialLookupMetrics; + LookupLevels.LookupContext context = + currentMetrics == null ? null : new LookupLevels.LookupContext(); + try { + return lookup(partition, bucket, key, context); + } finally { + if (currentMetrics != null) { + currentMetrics.reportLookup(context != null && context.remoteAccessed()); + } + } + } + + @Nullable + private InternalRow lookup( + BinaryRow partition, + int bucket, + InternalRow key, + @Nullable LookupLevels.LookupContext context) + throws IOException { + Map buckets = tableView.get(partition); + if (buckets == null || buckets.isEmpty()) { + return null; + } + BucketLookupState state = buckets.get(bucket); + if (state == null) { + return null; + } + + state.lock.readLock().lock(); + try { + LookupLevels lookupLevels = state.lookupLevels; + if (lookupLevels == null) { + return null; + } + + KeyValue kv = lookupLevels.lookup(key, startLevel, context); + if (kv == null || kv.valueKind().isRetract()) { + return null; + } else { + return kv.value(); + } + } finally { + state.lock.readLock().unlock(); + } + } + + @Override + public PaimonLocalTableQuery withValueProjection(int[] projection) { + this.readerFactoryBuilder.withReadValueType(rowType.project(projection)); + return this; + } + + public PaimonLocalTableQuery withIOManager(IOManager ioManager) { + this.ioManager = ioManager; + return this; + } + + public PaimonLocalTableQuery withCacheRowFilter(Filter cacheRowFilter) { + this.cacheRowFilter = cacheRowFilter; + return this; + } + + public PaimonLocalTableQuery withMetrics(@Nullable PartialLookupMetrics metrics) { + this.partialLookupMetrics = metrics; + return this; + } + + @Override + public InternalRowSerializer createValueSerializer() { + return InternalSerializers.create(readerFactoryBuilder.readValueType()); + } + + @Override + public void close() throws IOException { + // ConcurrentHashMap iteration is weakly consistent. close is expected not to race with + // refreshFiles for the same query instance; callers may rebuild this query after close. + for (Map.Entry> buckets : tableView.entrySet()) { + for (Map.Entry bucket : buckets.getValue().entrySet()) { + BucketLookupState state = bucket.getValue(); + state.lock.writeLock().lock(); + try { + if (state.lookupLevels != null) { + state.lookupLevels.close(); + } + } finally { + state.lock.writeLock().unlock(); + } + } + } + lookupFileCache.invalidateAll(); + tableView.clear(); + } + + private static class BucketLookupState { + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + + @Nullable private LookupLevels lookupLevels; + } +} diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/SharedLookupFileCache.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/SharedLookupFileCache.java new file mode 100644 index 00000000000..1e99d914c83 --- /dev/null +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/SharedLookupFileCache.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.lake.paimon.lookup; + +import org.apache.fluss.annotation.Internal; + +import org.apache.paimon.mergetree.LookupFile; +import org.apache.paimon.options.MemorySize; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Caffeine; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.RemovalCause; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.time.Duration; +import java.util.Objects; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; +import static org.apache.paimon.mergetree.LookupUtils.fileKibiBytes; + +/** A weighted lookup-file cache shared by multiple Paimon table lookupers. */ +@Internal +public final class SharedLookupFileCache implements AutoCloseable { + + private final Cache cache; + + /** Creates a shared lookup-file cache. */ + public SharedLookupFileCache(Duration fileRetention, MemorySize maxDiskSize) { + checkNotNull(fileRetention, "fileRetention must not be null."); + checkNotNull(maxDiskSize, "maxDiskSize must not be null."); + this.cache = + Caffeine.newBuilder() + .expireAfterAccess(fileRetention) + .maximumWeight(Math.max(1L, maxDiskSize.getKibiBytes())) + .weigher( + (Key key, LookupFile lookupFile) -> + Math.max(1, fileKibiBytes(lookupFile.localFile()))) + .removalListener(SharedLookupFileCache::removeLookupFile) + .executor(Runnable::run) + .build(); + } + + Cache namespaced(String namespace) { + return new NamespacedLookupFileCache(cache, namespace); + } + + /** Updates the maximum cache weight. */ + public void updateMaxDiskSize(MemorySize maxDiskSize) { + cache.policy().eviction().get().setMaximum(Math.max(1L, maxDiskSize.getKibiBytes())); + } + + @Override + public void close() { + cache.invalidateAll(); + cache.cleanUp(); + } + + private static void removeLookupFile( + @Nullable Key key, @Nullable LookupFile lookupFile, RemovalCause cause) { + if (lookupFile != null) { + try { + lookupFile.close(cause); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + static final class Key { + final String namespace; + final String fileName; + + Key(String namespace, String fileName) { + this.namespace = checkNotNull(namespace, "namespace must not be null."); + this.fileName = checkNotNull(fileName, "fileName must not be null."); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Key)) { + return false; + } + Key key = (Key) o; + return namespace.equals(key.namespace) && fileName.equals(key.fileName); + } + + @Override + public int hashCode() { + return Objects.hash(namespace, fileName); + } + } +} diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java index 9dfb4092489..4aeb8fc1615 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java @@ -25,7 +25,7 @@ import org.apache.fluss.exception.DiskWriteLockedException; import org.apache.fluss.exception.KvStorageException; import org.apache.fluss.exception.RetriableException; -import org.apache.fluss.lake.lakestorage.LakeStorage.LookuperContext; +import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.lake.lakestorage.TestingLakeCatalogContext; import org.apache.fluss.lake.paimon.PaimonLakeCatalog; @@ -75,16 +75,21 @@ import java.io.File; import java.io.IOException; import java.lang.reflect.Field; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.apache.fluss.config.ConfigOptions.KV_FORMAT_VERSION_2; import static org.apache.fluss.lake.paimon.utils.PaimonConversions.toPaimon; @@ -113,6 +118,7 @@ class PaimonLakeTableLookuperTest { private Configuration paimonConfig; private PaimonLakeCatalog lakeCatalog; private Catalog paimonCatalog; + private LakeTableLookupRuntime lookupRuntime; @BeforeEach void setUp() { @@ -122,10 +128,17 @@ void setUp() { paimonCatalog = CatalogFactory.createCatalog( CatalogContext.create(Options.fromMap(paimonConfig.toMap()))); + lookupRuntime = + new PaimonLakeStorage(paimonConfig) + .createLakeTableLookupRuntime( + tempWarehouseDir.getAbsolutePath(), LOOKUP_CACHE_MAX_DISK_BYTES); } @AfterEach void tearDown() throws Exception { + if (lookupRuntime != null) { + lookupRuntime.close(); + } if (paimonCatalog != null) { paimonCatalog.close(); } @@ -324,13 +337,8 @@ void testConcurrentFirstLookupsForDifferentPartitions() throws Exception { }; try (LakeTableLookuper lookuper = - new PaimonLakeTableLookuper( - paimonConfig, - tablePath, - tempWarehouseDir.getAbsolutePath(), - tableConfig(KvFormat.COMPACTED, 1, LakeLookupMode.SST), - LOOKUP_CACHE_MAX_DISK_BYTES, - diskWriteGuard)) { + createLookuper( + LakeLookupMode.SST, tablePath, KvFormat.COMPACTED, 1, diskWriteGuard)) { Future firstLookup = executor.submit( () -> @@ -453,6 +461,83 @@ void testDiskWriteLockBlocksOnlyLookupFileDownloads(LakeLookupMode lookupMode) } } + @Test + void testSharesIOManagerAndDeletesOnlyClosedLookuperFiles() throws Exception { + Schema schema = pkSchema(); + TablePath firstTablePath = TablePath.of(DB, "shared_io_first"); + TablePath secondTablePath = TablePath.of(DB, "shared_io_second"); + FileStoreTable firstTable = + createPaimonTable(firstTablePath, partitionedPkDescriptor(schema)); + FileStoreTable secondTable = + createPaimonTable(secondTablePath, partitionedPkDescriptor(schema)); + writeAndCommitData( + firstTable, + Collections.singletonMap( + 0, Collections.singletonList(paimonRow(1, "20240101", "Alice")))); + writeAndCommitData( + secondTable, + Collections.singletonMap( + 0, Collections.singletonList(paimonRow(2, "20240101", "Bob")))); + + File lookupDir = new File(tempWarehouseDir, "shared-lookup-cache"); + LakeTableLookupRuntime.Context firstLookuperContext = + new LakeTableLookupRuntime.Context( + paimonConfig, + "first-table", + tableConfig(KvFormat.COMPACTED, 1, LakeLookupMode.SST), + NO_OP_DISK_WRITE_GUARD); + LakeTableLookupRuntime.Context secondLookuperContext = + new LakeTableLookupRuntime.Context( + paimonConfig, + "second-table", + tableConfig(KvFormat.COMPACTED, 1, LakeLookupMode.SST), + NO_OP_DISK_WRITE_GUARD); + LakeTableLookupRuntime sharedLookupRuntime = + new PaimonLakeStorage(paimonConfig) + .createLakeTableLookupRuntime( + lookupDir.getAbsolutePath(), LOOKUP_CACHE_MAX_DISK_BYTES); + try { + try (LakeTableLookuper firstLookuper = + sharedLookupRuntime.createLakeTableLookuper( + firstTablePath, firstLookuperContext); + LakeTableLookuper secondLookuper = + sharedLookupRuntime.createLakeTableLookuper( + secondTablePath, secondLookuperContext)) { + assertThat( + firstLookuper.lookup( + paimonKey(schema, 1, "20240101"), + lookupContext(schema, "20240101", 0, SCHEMA_ID))) + .isNotNull(); + Set firstLookupFiles = regularFiles(lookupDir); + assertThat(firstLookupFiles).isNotEmpty(); + + assertThat( + secondLookuper.lookup( + paimonKey(schema, 2, "20240101"), + lookupContext(schema, "20240101", 0, SCHEMA_ID))) + .isNotNull(); + Set secondLookupFiles = regularFiles(lookupDir); + secondLookupFiles.removeAll(firstLookupFiles); + assertThat(secondLookupFiles).isNotEmpty(); + assertThat(lookupDir.listFiles(File::isDirectory)).hasSize(1); + + firstLookuper.close(); + assertThat(firstLookupFiles).allMatch(path -> !Files.exists(path)); + assertThat(secondLookupFiles).allMatch(Files::exists); + assertThat( + secondLookuper.lookup( + paimonKey(schema, 2, "20240101"), + lookupContext(schema, "20240101", 0, SCHEMA_ID))) + .isNotNull(); + } + assertThat(regularFiles(lookupDir)).isEmpty(); + assertThat(lookupDir.listFiles(File::isDirectory)).hasSize(1); + } finally { + sharedLookupRuntime.close(); + } + assertThat(lookupDir.listFiles(File::isDirectory)).isEmpty(); + } + @ParameterizedTest(name = "lookupMode={0}") @EnumSource(LakeLookupMode.class) void testLookupPartitionsWithSameHashCode(LakeLookupMode lookupMode) throws Exception { @@ -981,14 +1066,8 @@ private LakeTableLookuper createLookuper( KvFormat kvFormat, int kvFormatVersion, Runnable diskWriteGuard) { - TableConfig tableConfig = tableConfig(kvFormat, kvFormatVersion, lookupMode); - LookuperContext context = - new LookuperContext( - tempWarehouseDir.getAbsolutePath(), - tableConfig, - LOOKUP_CACHE_MAX_DISK_BYTES, - diskWriteGuard); - return new PaimonLakeStorage(paimonConfig).createLakeTableLookuper(tablePath, context); + return createLookuper( + tablePath, tableConfig(kvFormat, kvFormatVersion, lookupMode), diskWriteGuard); } private FileStoreTable createCompactionTable(TablePath tablePath, Schema schema) @@ -1068,6 +1147,23 @@ private static TableConfig tableConfig( return new TableConfig(config); } + private LakeTableLookuper createLookuper( + TablePath tablePath, TableConfig tableConfig, Runnable diskWriteGuard) { + return lookupRuntime.createLakeTableLookuper( + tablePath, + new LakeTableLookupRuntime.Context( + paimonConfig, tablePath.toString(), tableConfig, diskWriteGuard)); + } + + private static Set regularFiles(File directory) throws IOException { + if (!directory.exists()) { + return new HashSet<>(); + } + try (Stream paths = Files.walk(directory.toPath())) { + return paths.filter(Files::isRegularFile).collect(Collectors.toSet()); + } + } + private static Schema pkSchema() { return Schema.newBuilder() .column("id", DataTypes.INT()) diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/SharedLookupFileCacheTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/SharedLookupFileCacheTest.java new file mode 100644 index 00000000000..fa497dd3353 --- /dev/null +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/SharedLookupFileCacheTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.lake.paimon.lookup; + +import org.apache.paimon.lookup.LookupStoreReader; +import org.apache.paimon.mergetree.LookupFile; +import org.apache.paimon.options.MemorySize; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.time.Duration; +import java.util.Arrays; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link SharedLookupFileCache}. */ +class SharedLookupFileCacheTest { + + @TempDir private File tempDir; + + @Test + void testNamespaceIsolationAndGlobalLimit() throws Exception { + File firstFile = lookupFile("first.lookup"); + File secondFile = lookupFile("second.lookup"); + File thirdFile = lookupFile("third.lookup"); + + try (SharedLookupFileCache sharedCache = + new SharedLookupFileCache(Duration.ofHours(1), MemorySize.ofKibiBytes(2))) { + Cache firstNamespace = sharedCache.namespaced("first"); + Cache secondNamespace = sharedCache.namespaced("second"); + LookupFile firstLookupFile = lookupFile(firstFile); + LookupFile secondLookupFile = lookupFile(secondFile); + + firstNamespace.put("same-file-name", firstLookupFile); + secondNamespace.put("same-file-name", secondLookupFile); + assertThat(firstNamespace.getIfPresent("same-file-name")).isSameAs(firstLookupFile); + assertThat(secondNamespace.getIfPresent("same-file-name")).isSameAs(secondLookupFile); + + firstNamespace.invalidateAll(); + assertThat(firstFile).doesNotExist(); + assertThat(secondFile).exists(); + + sharedCache.updateMaxDiskSize(MemorySize.ofKibiBytes(1)); + secondNamespace.put("third-file", lookupFile(thirdFile)); + assertThat(Arrays.asList(secondFile, thirdFile).stream().filter(File::exists).count()) + .isLessThanOrEqualTo(1L); + } + + assertThat(secondFile).doesNotExist(); + assertThat(thirdFile).doesNotExist(); + } + + private File lookupFile(String name) throws IOException { + File file = new File(tempDir, name); + try (RandomAccessFile randomAccessFile = new RandomAccessFile(file, "rw")) { + randomAccessFile.setLength(1024L); + } + return file; + } + + private static LookupFile lookupFile(File file) { + return new LookupFile(file, 1, 0L, "v1", new NoOpLookupStoreReader(), () -> {}); + } + + private static final class NoOpLookupStoreReader implements LookupStoreReader { + @Override + public byte[] lookup(byte[] key) { + return null; + } + + @Override + public void close() {} + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java index d52ea903d30..c1a270ae4b0 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java @@ -26,6 +26,7 @@ import org.apache.fluss.lake.lakestorage.LakeStorage; import org.apache.fluss.lake.lakestorage.LakeStoragePlugin; import org.apache.fluss.lake.lakestorage.LakeStoragePluginSetUp; +import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.LakeLookupMode; @@ -66,6 +67,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Stream; import static org.apache.fluss.server.utils.LakeStorageUtils.extractLakeProperties; @@ -84,8 +86,12 @@ * closed after its last lookup releases it. A new required lake snapshot refreshes the cached * lookuper in place. * - *

Up to ten table lookupers are cached. Each lookuper receives one tenth of the server-level - * disk budget, and Caffeine evicts lookupers when the table limit is exceeded. + *

One lake-format-specific lookup runtime is initialized with this manager, or when Paimon is + * configured dynamically, and shared by all table lookupers. The runtime owns TabletServer-scoped + * resources such as Paimon's I/O manager. + * + *

Up to ten table lookupers are cached. Their lookup files share one server-level disk budget, + * and Caffeine evicts lookupers when the table limit is exceeded. * *

Historical lookup cache I/O participates in TabletServer disk write protection. Existing cache * hits remain available when the data disk is write-locked, while lookups that need to download new @@ -105,15 +111,14 @@ class HistoricalLakeLookupManager implements AutoCloseable { private static final String LOOKUP_CACHE_DISK_SIZE_TASK_NAME = "historical-lookup-cache-disk-size"; private static final Duration LOOKUP_CACHE_DISK_SIZE_CHECK_INTERVAL = Duration.ofMinutes(3); - // TODO: Share one Paimon IOManager disk budget across all table lookupers and evict cached - // entries by data file instead of reserving fixed per-table capacity. See - // https://github.com/apache/fluss/issues/3955. private static final int MAX_CACHED_TABLES = 10; private volatile Configuration conf; private volatile long lakeConfigVersion; private final @Nullable PluginManager pluginManager; + private volatile @Nullable LakeTableLookupRuntime lookupRuntime; private final Counter capacityEvictions; + private final AtomicLong lookuperIdSequence; private final Cache lakeTableLookupers; private final ConcurrentMap requiredLakeSnapshotIds = new ConcurrentHashMap<>(); private final File historicalLookupCacheRootDir; @@ -123,7 +128,7 @@ class HistoricalLakeLookupManager implements AutoCloseable { // the cache to grow back to the maximum ratio after disk usage recovers. private final Runnable diskWriteGuard; - private volatile long lookupCacheMaxDiskBytesPerTable; + private volatile long lookupCacheMaxDiskBytes; private volatile long lookupCacheDiskSize; private volatile boolean started; @@ -164,8 +169,8 @@ class HistoricalLakeLookupManager implements AutoCloseable { checkArgument(dataDirVolumeBytes > 0, "dataDirVolumeBytes must be greater than 0."); this.dataDirVolumeBytes = dataDirVolumeBytes; this.diskWriteGuard = checkNotNull(diskWriteGuard, "diskWriteGuard must not be null."); - this.lookupCacheMaxDiskBytesPerTable = - cacheBytesPerTable( + this.lookupCacheMaxDiskBytes = + cacheBytes( conf.get( ConfigOptions .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO)); @@ -182,6 +187,8 @@ class HistoricalLakeLookupManager implements AutoCloseable { .executor(Runnable::run) .removalListener(this::onLookuperRemoved) .build(); + this.lookuperIdSequence = new AtomicLong(); + this.lookupRuntime = createLookupRuntime(conf); } private static com.github.benmanes.caffeine.cache.Scheduler createCacheScheduler( @@ -266,6 +273,12 @@ public void close() { lakeTableLookupers.invalidateAll(); lakeTableLookupers.cleanUp(); requiredLakeSnapshotIds.clear(); + LakeTableLookupRuntime runtime; + synchronized (this) { + runtime = lookupRuntime; + lookupRuntime = null; + } + IOUtils.closeQuietly(runtime, "historical lake lookup runtime"); } /** Invalidates the cached lake lookuper for the given table. */ @@ -289,6 +302,16 @@ Counter capacityEvictions() { return capacityEvictions; } + @VisibleForTesting + boolean hasLookupRuntime() { + return lookupRuntime != null; + } + + @VisibleForTesting + long lookupCacheMaxDiskBytes() { + return lookupCacheMaxDiskBytes; + } + /** Applies dynamic historical lookup configuration changes. */ void reconfigure(Configuration newConf) { checkNotNull(newConf, "newConf must not be null."); @@ -300,13 +323,13 @@ void reconfigure(Configuration newConf) { ConfigOptions .SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS); synchronized (this) { - long newMaxBytesPerTable = - cacheBytesPerTable( + long newMaxDiskBytes = + cacheBytes( newConf.get( ConfigOptions .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO)); - cacheLimitChanged = newMaxBytesPerTable != lookupCacheMaxDiskBytesPerTable; - lookupCacheMaxDiskBytesPerTable = newMaxBytesPerTable; + cacheLimitChanged = newMaxDiskBytes != lookupCacheMaxDiskBytes; + lookupCacheMaxDiskBytes = newMaxDiskBytes; lakeConfigChanged = hasLakeConfigChanged(conf, newConf); expirationChanged = @@ -314,6 +337,12 @@ void reconfigure(Configuration newConf) { conf.get( ConfigOptions .SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS)); + if (lakeConfigChanged && lookupRuntime == null) { + lookupRuntime = createLookupRuntime(newConf); + } + if (cacheLimitChanged && lookupRuntime != null) { + lookupRuntime.updateLookupCacheMaxDiskBytes(newMaxDiskBytes); + } // Publish the configuration before its version. A lookup that observes the new version // must also observe the matching configuration snapshot. conf = newConf; @@ -328,12 +357,11 @@ void reconfigure(Configuration newConf) { .get() .setExpiresAfter(newExpiration.toMillis(), TimeUnit.MILLISECONDS); } - if (lakeConfigChanged || cacheLimitChanged) { + if (lakeConfigChanged) { // Do not invalidate while holding this monitor: lookuper creation holds a cache key // lock before preparing the lookup directory under the same monitor. Invalidation // closes inactive lookupers immediately and active lookupers after their last lookup - // releases them. After a cache limit change, the next lookup creates a Paimon lookuper - // with the updated per-table limit. + // releases them. lakeTableLookupers.invalidateAll(); lakeTableLookupers.cleanUp(); } @@ -382,9 +410,8 @@ private LookupContext createLookupContext( LakeTableLookuper createLakeTableLookuper( TablePath tablePath, - String ioTmpDir, TableConfig tableConfig, - long cacheSizeBytes, + String cacheNamespace, Configuration clusterConf) { DataLakeFormat dataLakeFormat = clusterConf.get(ConfigOptions.DATALAKE_FORMAT); if (dataLakeFormat == null) { @@ -404,14 +431,32 @@ LakeTableLookuper createLakeTableLookuper( "Historical lookup requires cluster lake storage properties to be configured."); } + LakeTableLookupRuntime runtime = lookupRuntime; + if (runtime == null) { + throw new LakeStorageNotConfiguredException( + "Historical lake lookup runtime has not been initialized."); + } + return runtime.createLakeTableLookuper( + tablePath, + new LakeTableLookupRuntime.Context( + Configuration.fromMap(lakeProperties), + cacheNamespace, + tableConfig, + diskWriteGuard)); + } + + private @Nullable LakeTableLookupRuntime createLookupRuntime(Configuration configuration) { + DataLakeFormat dataLakeFormat = configuration.get(ConfigOptions.DATALAKE_FORMAT); + Map lakeProperties = extractLakeProperties(configuration); + if (dataLakeFormat != DataLakeFormat.PAIMON || lakeProperties == null) { + return null; + } LakeStoragePlugin lakeStoragePlugin = LakeStoragePluginSetUp.fromDataLakeFormat(dataLakeFormat.toString(), pluginManager); LakeStorage lakeStorage = lakeStoragePlugin.createLakeStorage(Configuration.fromMap(lakeProperties)); - return lakeStorage.createLakeTableLookuper( - tablePath, - new LakeStorage.LookuperContext( - ioTmpDir, tableConfig, cacheSizeBytes, diskWriteGuard)); + return lakeStorage.createLakeTableLookupRuntime( + historicalLookupCacheRootDir.getAbsolutePath(), lookupCacheMaxDiskBytes); } private static boolean hasLakeConfigChanged(Configuration currentConf, Configuration newConf) { @@ -421,11 +466,16 @@ private static boolean hasLakeConfigChanged(Configuration currentConf, Configura extractLakeProperties(currentConf), extractLakeProperties(newConf)); } - private long cacheBytesPerTable(double ratio) { + private long cacheBytes(double ratio) { checkArgument(ratio > 0.0 && ratio <= 1.0, "ratio must be within (0.0, 1.0]."); long totalCacheBytes = Math.min(dataDirVolumeBytes, (long) Math.ceil(dataDirVolumeBytes * ratio)); - return Math.max(1L, totalCacheBytes / MAX_CACHED_TABLES); + return Math.max(1L, totalCacheBytes); + } + + private static String cacheNamespace( + long tableId, int schemaId, long lakeConfigVersion, long lookuperId) { + return tableId + "-" + schemaId + "-" + lakeConfigVersion + "-" + lookuperId; } /** Returns the most recently sampled historical lookup cache footprint, in bytes. */ @@ -461,34 +511,12 @@ private static long fileSize(Path path) { } private static void closeLookuper(CachedLakeTableLookuper cachedLookuper) { - closeLookuper(cachedLookuper.lookuper, cachedLookuper.tableLookupDir); - } - - private static void closeLookuper(LakeTableLookuper lookuper, File tableLookupDir) { - try { - IOUtils.closeQuietly(lookuper, "historical lake table lookuper"); - } finally { - deleteTableLookupDirIfEmpty(tableLookupDir); - } - } - - private static void deleteTableLookupDirIfEmpty(File tableLookupDir) { - if (FileUtils.isDirectoryEmpty(tableLookupDir)) { - try { - Files.deleteIfExists(tableLookupDir.toPath()); - } catch (IOException e) { - LOG.debug( - "Failed to delete empty historical lookup directory {}.", - tableLookupDir, - e); - } - } + IOUtils.closeQuietly(cachedLookuper.lookuper, "historical lake table lookuper"); } private CachedLakeTableLookuper acquireLookuper(LookupContext context, TableInfo tableInfo) { long currentLakeConfigVersion = lakeConfigVersion; Configuration currentConf = conf; - long cacheSizeBytes = lookupCacheMaxDiskBytesPerTable; LakeLookupMode lookupMode = tableInfo.getTableConfig().getHistoricalLookupMode(); return lakeTableLookupers .asMap() @@ -502,26 +530,22 @@ private CachedLakeTableLookuper acquireLookuper(LookupContext context, TableInfo requiredLakeSnapshotIds.get(context.tableId); CachedLakeTableLookuper selectedLookuper = currentLookuper; // Create the lookuper lazily, and recreate it after schema, - // lookup mode, lake configuration, or server cache size changes so it - // reloads lake table/query state and uses the current - // settings. + // lookup mode, or lake configuration changes so it reloads lake + // table/query state and uses the current settings. if (selectedLookuper == null || selectedLookuper.schemaId != context.schemaId || selectedLookuper.lookupMode != lookupMode || selectedLookuper.lakeConfigVersion - != currentLakeConfigVersion - || selectedLookuper.cacheSizeBytes != cacheSizeBytes) { - File tableLookupDir = - FlussPaths.historicalLookupTableDir( - historicalLookupCacheRootDir, - context.tablePath, - context.tableId); + != currentLakeConfigVersion) { LakeTableLookuper lookuper = createLakeTableLookuper( context.tablePath, - tableLookupDir.getAbsolutePath(), tableInfo.getTableConfig(), - cacheSizeBytes, + cacheNamespace( + context.tableId, + context.schemaId, + currentLakeConfigVersion, + lookuperIdSequence.getAndIncrement()), currentConf); selectedLookuper = new CachedLakeTableLookuper( @@ -530,9 +554,7 @@ private CachedLakeTableLookuper acquireLookuper(LookupContext context, TableInfo context.schemaId, lookupMode, currentLakeConfigVersion, - cacheSizeBytes, requiredLakeSnapshotId, - tableLookupDir, lookuper); } // Pin the lookuper before leaving the atomic cache update. @@ -567,11 +589,9 @@ private static final class CachedLakeTableLookuper { private final int schemaId; private final LakeLookupMode lookupMode; private final long lakeConfigVersion; - private final long cacheSizeBytes; /** The opaque lake snapshot ID covered by the last file refresh, or null if none. */ private @Nullable Long lakeSnapshotId; - private final File tableLookupDir; private final LakeTableLookuper lookuper; private int activeLookups; private boolean invalidated; @@ -583,18 +603,14 @@ private CachedLakeTableLookuper( int schemaId, LakeLookupMode lookupMode, long lakeConfigVersion, - long cacheSizeBytes, @Nullable Long lakeSnapshotId, - File tableLookupDir, LakeTableLookuper lookuper) { this.tableId = tableId; this.tablePath = tablePath; this.schemaId = schemaId; this.lookupMode = lookupMode; this.lakeConfigVersion = lakeConfigVersion; - this.cacheSizeBytes = cacheSizeBytes; this.lakeSnapshotId = lakeSnapshotId; - this.tableLookupDir = tableLookupDir; this.lookuper = lookuper; } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/lakehouse/TestingPaimonStoragePlugin.java b/fluss-server/src/test/java/org/apache/fluss/server/lakehouse/TestingPaimonStoragePlugin.java index 88ad329e04f..ca30f502291 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/lakehouse/TestingPaimonStoragePlugin.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/lakehouse/TestingPaimonStoragePlugin.java @@ -28,6 +28,8 @@ import org.apache.fluss.lake.lakestorage.LakeCatalog; import org.apache.fluss.lake.lakestorage.LakeStorage; import org.apache.fluss.lake.lakestorage.LakeStoragePlugin; +import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime; +import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.lake.serializer.SimpleVersionedSerializer; import org.apache.fluss.lake.source.LakeSource; import org.apache.fluss.lake.writer.LakeTieringFactory; @@ -86,6 +88,24 @@ public LakeCatalog createLakeCatalog() { public LakeSource createLakeSource(TablePath tablePath) { throw new UnsupportedOperationException("Not implemented"); } + + @Override + public LakeTableLookupRuntime createLakeTableLookupRuntime( + String ioTmpDir, long lookupCacheMaxDiskBytes) { + return new LakeTableLookupRuntime() { + @Override + public LakeTableLookuper createLakeTableLookuper( + TablePath tablePath, Context context) { + throw new UnsupportedOperationException("Not implemented"); + } + + @Override + public void updateLookupCacheMaxDiskBytes(long lookupCacheMaxDiskBytes) {} + + @Override + public void close() {} + }; + } } /** Paimon implementation of LakeCatalog for testing purpose. */ diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java index 37e03ddc48c..beecbe57661 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java @@ -82,7 +82,6 @@ void testCleansAndCreatesLookupCacheDirectoryOnStartup() throws Exception { assertThat(staleLookupFile).doesNotExist(); assertThat(serverLookupDir).isDirectory(); lookup(manager, PARTITION_TABLE_INFO); - assertThat(manager.createdIoTmpDirs.get(0)).startsWith(serverLookupDir.getAbsolutePath()); File liveLookupFile = new File(serverLookupDir, "live-lookup-file"); assertThat(liveLookupFile.createNewFile()).isTrue(); @@ -332,11 +331,37 @@ void testEvictsLookuperWhenCachedTableLimitIsExceeded() throws Exception { assertThat(manager.createdLookupers).hasSize(11); assertThat(manager.createdLookupers).filteredOn(lookuper -> lookuper.closed).hasSize(1); - assertThat(manager.createdCacheSizes).containsOnly(2L); + assertThat(manager.createdCacheNamespaces).doesNotHaveDuplicates(); + assertThat(manager.lookupCacheMaxDiskBytes()).isEqualTo(20L); assertThat(manager.cachedTableCount()).isEqualTo(10); assertThat(manager.capacityEvictions().getCount()).isEqualTo(1); } + @Test + void testUpdatesSharedCacheLimitWithoutReplacingLookuper() throws Exception { + Configuration initialConf = conf(); + initialConf.set( + ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, 0.10); + TestingHistoricalLakeLookupManager manager = + new TestingHistoricalLakeLookupManager( + initialConf, + Ticker.systemTicker(), + Scheduler.disabledScheduler(), + 100L, + 0L); + manager.startup(NO_OP_SCHEDULER); + lookup(manager, PARTITION_TABLE_INFO); + TestingLakeTableLookuper lookuper = manager.createdLookupers.get(0); + + Configuration newConf = new Configuration(initialConf); + newConf.set(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, 0.20); + manager.reconfigure(newConf); + + assertThat(manager.lookupCacheMaxDiskBytes()).isEqualTo(20L); + assertThat(lookuper.closed).isFalse(); + assertThat(manager.cachedTableCount()).isOne(); + } + @Test void testReconfiguresLakePropertiesAndInvalidatesLookuper() throws Exception { Configuration initialConf = conf(); @@ -344,6 +369,7 @@ void testReconfiguresLakePropertiesAndInvalidatesLookuper() throws Exception { initialConf.setString("datalake.paimon.warehouse", "old-warehouse"); TestingHistoricalLakeLookupManager manager = new TestingHistoricalLakeLookupManager(initialConf); + assertThat(manager.hasLookupRuntime()).isTrue(); manager.startup(NO_OP_SCHEDULER); lookup(manager, PARTITION_TABLE_INFO); @@ -420,9 +446,8 @@ private static final class TestingHistoricalLakeLookupManager extends HistoricalLakeLookupManager { private final List createdLookupers = new ArrayList<>(); private final List createdTablePaths = new ArrayList<>(); - private final List createdIoTmpDirs = new ArrayList<>(); private final List createdTableConfigs = new ArrayList<>(); - private final List createdCacheSizes = new ArrayList<>(); + private final List createdCacheNamespaces = new ArrayList<>(); private final List createdClusterConfigs = new ArrayList<>(); private final long lookupCacheFileBytes; @@ -471,17 +496,18 @@ private TestingHistoricalLakeLookupManager( @Override LakeTableLookuper createLakeTableLookuper( TablePath tablePath, - String ioTmpDir, TableConfig tableConfig, - long cacheSizeBytes, + String cacheNamespace, Configuration clusterConf) { TestingLakeTableLookuper lookuper = - new TestingLakeTableLookuper(new File(ioTmpDir), lookupCacheFileBytes); + new TestingLakeTableLookuper( + FlussPaths.historicalLookupRootDir( + new File(clusterConf.get(ConfigOptions.DATA_DIR))), + lookupCacheFileBytes); createdLookupers.add(lookuper); createdTablePaths.add(tablePath); - createdIoTmpDirs.add(ioTmpDir); createdTableConfigs.add(tableConfig); - createdCacheSizes.add(cacheSizeBytes); + createdCacheNamespaces.add(cacheNamespace); createdClusterConfigs.add(clusterConf); return lookuper; } From ff9a2888d06bbab3a8afb7bfcd9cc63b00ebb43b Mon Sep 17 00:00:00 2001 From: zhangjunfan Date: Mon, 21 Sep 2026 18:01:14 +0800 Subject: [PATCH 2/8] introduce runtimeOption --- .../fluss/lake/lakestorage/LakeStorage.java | 5 ++- .../lakestorage/LakeTableLookupRuntime.java | 44 +++++++++++++++++-- .../lakestorage/PluginLakeStorageWrapper.java | 10 ++--- .../lake/lakestorage/LakeStorageTest.java | 8 ++-- .../fluss/lake/paimon/PaimonLakeStorage.java | 25 +++++------ .../lookup/PaimonLakeTableLookuperTest.java | 13 ++++-- .../HistoricalLakeLookupManager.java | 6 ++- .../lakehouse/TestingPaimonStoragePlugin.java | 5 ++- 8 files changed, 83 insertions(+), 33 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java index bba509188f6..03ac4cf4e8d 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java @@ -18,6 +18,7 @@ package org.apache.fluss.lake.lakestorage; import org.apache.fluss.annotation.PublicEvolving; +import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime.LookupRuntimeOptions; import org.apache.fluss.lake.source.LakeSource; import org.apache.fluss.lake.writer.LakeTieringFactory; import org.apache.fluss.metadata.TablePath; @@ -55,11 +56,11 @@ public interface LakeStorage { * Creates a TabletServer-scoped runtime for lake table point lookup. * * @param ioTmpDir local directory shared by lookupers for temporary files - * @param lookupCacheMaxDiskBytes maximum local lookup cache size in bytes + * @param options initial runtime resource settings * @return the lookup runtime */ default LakeTableLookupRuntime createLakeTableLookupRuntime( - String ioTmpDir, long lookupCacheMaxDiskBytes) { + String ioTmpDir, LookupRuntimeOptions options) { throw new UnsupportedOperationException( "Point lookup is not supported for this lake storage."); } diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookupRuntime.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookupRuntime.java index 2eb41b2ee26..8656e1710bd 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookupRuntime.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookupRuntime.java @@ -22,9 +22,10 @@ import org.apache.fluss.config.TableConfig; import org.apache.fluss.metadata.TablePath; +import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; -/** TabletServer-scoped runtime for creating lake table lookupers. */ +/** TabletServer-scoped runtime managing shared resources and creating lake table lookupers. */ @PublicEvolving public interface LakeTableLookupRuntime extends AutoCloseable { @@ -37,8 +38,45 @@ public interface LakeTableLookupRuntime extends AutoCloseable { */ LakeTableLookuper createLakeTableLookuper(TablePath tablePath, Context context); - /** Updates the maximum local lookup cache size in bytes. */ - void updateLookupCacheMaxDiskBytes(long lookupCacheMaxDiskBytes); + /** + * Applies a new snapshot of the runtime resource settings to existing and future lookupers + * without replacing them. + * + *

This method may be called concurrently with lookuper creation and lookups. Implementations + * must apply the settings in a thread-safe manner. Lake-format and table-specific configuration + * is outside the scope of this method. + * + * @param options the new runtime resource settings + */ + void reconfigure(LookupRuntimeOptions options); + + /** + * Immutable snapshot of the format-independent resource settings for a lookup runtime. + * + *

These settings apply to resources shared by all table lookupers in one runtime. + * Lake-format and table-specific configuration is supplied separately when creating a lookuper. + */ + final class LookupRuntimeOptions { + + private final long localCacheMaxBytes; + + /** + * Creates runtime resource settings. + * + * @param localCacheMaxBytes positive disk-space budget in bytes for local caches shared by + * all lookupers in the runtime; implementations without local disk caches may ignore + * this budget + */ + public LookupRuntimeOptions(long localCacheMaxBytes) { + checkArgument(localCacheMaxBytes > 0, "localCacheMaxBytes must be greater than 0."); + this.localCacheMaxBytes = localCacheMaxBytes; + } + + /** Returns the runtime-wide disk-space budget for local caches, in bytes. */ + public long localCacheMaxBytes() { + return localCacheMaxBytes; + } + } /** Runtime context for creating a lake table lookuper. */ final class Context { diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java index 2a81bdda7d9..42916ee9638 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java @@ -20,6 +20,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.TableAlreadyExistException; import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime.LookupRuntimeOptions; import org.apache.fluss.lake.source.LakeSource; import org.apache.fluss.lake.writer.LakeTieringFactory; import org.apache.fluss.metadata.TableChange; @@ -140,11 +141,10 @@ public LakeSource createLakeSource(TablePath tablePath) { @Override public LakeTableLookupRuntime createLakeTableLookupRuntime( - String ioTmpDir, long lookupCacheMaxDiskBytes) { + String ioTmpDir, LookupRuntimeOptions options) { try (TemporaryClassLoaderContext ignored = TemporaryClassLoaderContext.of(loader)) { return new ClassLoaderFixingLakeTableLookupRuntime( - inner.createLakeTableLookupRuntime(ioTmpDir, lookupCacheMaxDiskBytes), - loader); + inner.createLakeTableLookupRuntime(ioTmpDir, options), loader); } } } @@ -170,9 +170,9 @@ public LakeTableLookuper createLakeTableLookuper(TablePath tablePath, Context co } @Override - public void updateLookupCacheMaxDiskBytes(long lookupCacheMaxDiskBytes) { + public void reconfigure(LookupRuntimeOptions options) { try (TemporaryClassLoaderContext ignored = TemporaryClassLoaderContext.of(loader)) { - inner.updateLookupCacheMaxDiskBytes(lookupCacheMaxDiskBytes); + inner.reconfigure(options); } } diff --git a/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java b/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java index 08f0d785c18..7374a937015 100644 --- a/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java @@ -20,6 +20,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.TableAlreadyExistException; import org.apache.fluss.exception.TableNotExistException; +import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime.LookupRuntimeOptions; import org.apache.fluss.lake.source.LakeSource; import org.apache.fluss.lake.writer.LakeTieringFactory; import org.apache.fluss.metadata.TableChange; @@ -98,7 +99,8 @@ void testWithPluginManager() throws Exception { .isInstanceOf(TestPaimonLakeCatalog.class); LakeTableLookupRuntime lookupRuntime = - lakeStorage.createLakeTableLookupRuntime("lookup-dir", 1024L); + lakeStorage.createLakeTableLookupRuntime( + "lookup-dir", new LookupRuntimeOptions(1024L)); assertThat(lookupRuntime) .isInstanceOf( PluginLakeStorageWrapper.ClassLoaderFixingLakeTableLookupRuntime.class); @@ -161,7 +163,7 @@ public LakeSource createLakeSource(TablePath tablePath) { @Override public LakeTableLookupRuntime createLakeTableLookupRuntime( - String ioTmpDir, long lookupCacheMaxDiskBytes) { + String ioTmpDir, LookupRuntimeOptions options) { return new TestLakeTableLookupRuntime(); } } @@ -176,7 +178,7 @@ public LakeTableLookuper createLakeTableLookuper(TablePath tablePath, Context co } @Override - public void updateLookupCacheMaxDiskBytes(long lookupCacheMaxDiskBytes) {} + public void reconfigure(LookupRuntimeOptions options) {} @Override public void close() { diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java index 5779444a920..bb853f3a45d 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java @@ -20,6 +20,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.lake.lakestorage.LakeStorage; import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime; +import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime.LookupRuntimeOptions; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.lake.paimon.lookup.PaimonLakeTableLookuper; import org.apache.fluss.lake.paimon.lookup.PaimonScanBasedTableLookuper; @@ -39,7 +40,6 @@ import org.apache.paimon.disk.IOManager; import org.apache.paimon.options.MemorySize; -import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; /** Paimon implementation of {@link LakeStorage}. */ @@ -68,8 +68,8 @@ public LakeSource createLakeSource(TablePath tablePath) { @Override public LakeTableLookupRuntime createLakeTableLookupRuntime( - String ioTmpDir, long lookupCacheMaxDiskBytes) { - return new PaimonLakeTableLookupRuntime(ioTmpDir, lookupCacheMaxDiskBytes); + String ioTmpDir, LookupRuntimeOptions options) { + return new PaimonLakeTableLookupRuntime(ioTmpDir, options); } /** Paimon lookup runtime sharing one I/O manager across table lookupers. */ @@ -77,22 +77,22 @@ private static final class PaimonLakeTableLookupRuntime implements LakeTableLook private final IOManager ioManager; private final SharedLookupFileCache lookupFileCache; - private PaimonLakeTableLookupRuntime(String ioTmpDir, long lookupCacheMaxDiskBytes) { - checkArgument( - lookupCacheMaxDiskBytes > 0, "lookupCacheMaxDiskBytes must be greater than 0."); + private PaimonLakeTableLookupRuntime(String ioTmpDir, LookupRuntimeOptions options) { + checkNotNull(options, "options must not be null."); this.ioManager = IOManager.create(checkNotNull(ioTmpDir, "ioTmpDir must not be null.")); - // ponytail: one runtime-wide retention; add a server option if this needs tuning. this.lookupFileCache = new SharedLookupFileCache( CoreOptions.LOOKUP_CACHE_FILE_RETENTION.defaultValue(), - new MemorySize(lookupCacheMaxDiskBytes)); + new MemorySize(options.localCacheMaxBytes())); } @Override public LakeTableLookuper createLakeTableLookuper(TablePath tablePath, Context context) { if (context.tableConfig().getHistoricalLookupMode() == LakeLookupMode.SCAN) { return new PaimonScanBasedTableLookuper( - new Configuration(context.lakeConfiguration()), tablePath, context.tableConfig()); + new Configuration(context.lakeConfiguration()), + tablePath, + context.tableConfig()); } return new PaimonLakeTableLookuper( new Configuration(context.lakeConfiguration()), @@ -105,10 +105,9 @@ public LakeTableLookuper createLakeTableLookuper(TablePath tablePath, Context co } @Override - public void updateLookupCacheMaxDiskBytes(long lookupCacheMaxDiskBytes) { - checkArgument( - lookupCacheMaxDiskBytes > 0, "lookupCacheMaxDiskBytes must be greater than 0."); - lookupFileCache.updateMaxDiskSize(new MemorySize(lookupCacheMaxDiskBytes)); + public void reconfigure(LookupRuntimeOptions options) { + checkNotNull(options, "options must not be null."); + lookupFileCache.updateMaxDiskSize(new MemorySize(options.localCacheMaxBytes())); } @Override diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java index 4aeb8fc1615..f89992db293 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java @@ -26,6 +26,7 @@ import org.apache.fluss.exception.KvStorageException; import org.apache.fluss.exception.RetriableException; import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime; +import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime.LookupRuntimeOptions; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.lake.lakestorage.TestingLakeCatalogContext; import org.apache.fluss.lake.paimon.PaimonLakeCatalog; @@ -131,7 +132,8 @@ void setUp() { lookupRuntime = new PaimonLakeStorage(paimonConfig) .createLakeTableLookupRuntime( - tempWarehouseDir.getAbsolutePath(), LOOKUP_CACHE_MAX_DISK_BYTES); + tempWarehouseDir.getAbsolutePath(), + new LookupRuntimeOptions(LOOKUP_CACHE_MAX_DISK_BYTES)); } @AfterEach @@ -338,7 +340,11 @@ void testConcurrentFirstLookupsForDifferentPartitions() throws Exception { try (LakeTableLookuper lookuper = createLookuper( - LakeLookupMode.SST, tablePath, KvFormat.COMPACTED, 1, diskWriteGuard)) { + LakeLookupMode.SST, + tablePath, + KvFormat.COMPACTED, + 1, + diskWriteGuard)) { Future firstLookup = executor.submit( () -> @@ -495,7 +501,8 @@ void testSharesIOManagerAndDeletesOnlyClosedLookuperFiles() throws Exception { LakeTableLookupRuntime sharedLookupRuntime = new PaimonLakeStorage(paimonConfig) .createLakeTableLookupRuntime( - lookupDir.getAbsolutePath(), LOOKUP_CACHE_MAX_DISK_BYTES); + lookupDir.getAbsolutePath(), + new LookupRuntimeOptions(LOOKUP_CACHE_MAX_DISK_BYTES)); try { try (LakeTableLookuper firstLookuper = sharedLookupRuntime.createLakeTableLookuper( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java index c1a270ae4b0..f1c469de1a5 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java @@ -27,6 +27,7 @@ import org.apache.fluss.lake.lakestorage.LakeStoragePlugin; import org.apache.fluss.lake.lakestorage.LakeStoragePluginSetUp; import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime; +import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime.LookupRuntimeOptions; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.LakeLookupMode; @@ -341,7 +342,7 @@ void reconfigure(Configuration newConf) { lookupRuntime = createLookupRuntime(newConf); } if (cacheLimitChanged && lookupRuntime != null) { - lookupRuntime.updateLookupCacheMaxDiskBytes(newMaxDiskBytes); + lookupRuntime.reconfigure(new LookupRuntimeOptions(newMaxDiskBytes)); } // Publish the configuration before its version. A lookup that observes the new version // must also observe the matching configuration snapshot. @@ -456,7 +457,8 @@ LakeTableLookuper createLakeTableLookuper( LakeStorage lakeStorage = lakeStoragePlugin.createLakeStorage(Configuration.fromMap(lakeProperties)); return lakeStorage.createLakeTableLookupRuntime( - historicalLookupCacheRootDir.getAbsolutePath(), lookupCacheMaxDiskBytes); + historicalLookupCacheRootDir.getAbsolutePath(), + new LookupRuntimeOptions(lookupCacheMaxDiskBytes)); } private static boolean hasLakeConfigChanged(Configuration currentConf, Configuration newConf) { diff --git a/fluss-server/src/test/java/org/apache/fluss/server/lakehouse/TestingPaimonStoragePlugin.java b/fluss-server/src/test/java/org/apache/fluss/server/lakehouse/TestingPaimonStoragePlugin.java index ca30f502291..9ce94b72cc8 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/lakehouse/TestingPaimonStoragePlugin.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/lakehouse/TestingPaimonStoragePlugin.java @@ -29,6 +29,7 @@ import org.apache.fluss.lake.lakestorage.LakeStorage; import org.apache.fluss.lake.lakestorage.LakeStoragePlugin; import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime; +import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime.LookupRuntimeOptions; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.lake.serializer.SimpleVersionedSerializer; import org.apache.fluss.lake.source.LakeSource; @@ -91,7 +92,7 @@ public LakeSource createLakeSource(TablePath tablePath) { @Override public LakeTableLookupRuntime createLakeTableLookupRuntime( - String ioTmpDir, long lookupCacheMaxDiskBytes) { + String ioTmpDir, LookupRuntimeOptions options) { return new LakeTableLookupRuntime() { @Override public LakeTableLookuper createLakeTableLookuper( @@ -100,7 +101,7 @@ public LakeTableLookuper createLakeTableLookuper( } @Override - public void updateLookupCacheMaxDiskBytes(long lookupCacheMaxDiskBytes) {} + public void reconfigure(LookupRuntimeOptions options) {} @Override public void close() {} From 90d74151965c27e2689907664b3daf2ebb563e92 Mon Sep 17 00:00:00 2001 From: zhangjunfan Date: Sat, 26 Sep 2026 21:22:12 +0800 Subject: [PATCH 3/8] fix all by yuxia --- LICENSE | 1 + .../apache/fluss/config/ConfigOptions.java | 6 +- .../fluss/lake/lakestorage/LakeStorage.java | 8 +- ...ime.java => LakeTableLookuperManager.java} | 32 ++- .../lakestorage/PluginLakeStorageWrapper.java | 27 +- .../org/apache/fluss/metrics/MetricNames.java | 4 +- .../lake/lakestorage/LakeStorageTest.java | 58 ++-- .../fluss/lake/paimon/PaimonLakeStorage.java | 23 +- .../lookup/NamespacedLookupFileCache.java | 28 +- .../paimon/lookup/PaimonLocalTableQuery.java | 9 +- .../paimon/lookup/SharedLookupFileCache.java | 31 ++- .../lookup/PaimonLakeTableLookuperTest.java | 43 +-- .../lookup/SharedLookupFileCacheTest.java | 154 ++++++++++- .../fluss/server/replica/ReplicaManager.java | 6 +- .../HistoricalLakeLookupManager.java | 247 ++++++++++-------- .../HistoricalPartitionManager.java | 7 +- .../lakehouse/TestingPaimonStoragePlugin.java | 8 +- .../HistoricalLakeLookupManagerTest.java | 131 +++++++++- website/docs/maintenance/configuration.md | 2 + .../observability/monitor-metrics.md | 6 +- 20 files changed, 617 insertions(+), 214 deletions(-) rename fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/{LakeTableLookupRuntime.java => LakeTableLookuperManager.java} (79%) diff --git a/LICENSE b/LICENSE index 199ba70b70d..2fe515bb8cd 100644 --- a/LICENSE +++ b/LICENSE @@ -368,6 +368,7 @@ Apache Kafka ./fluss-server/src/main/java/org/apache/fluss/server/utils/timer/TimingWheel.java Apache Paimon +./fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLocalTableQuery.java ./fluss-common/src/main/java/org/apache/fluss/predicate/And.java ./fluss-common/src/main/java/org/apache/fluss/predicate/CompareUtils.java ./fluss-common/src/main/java/org/apache/fluss/predicate/CompoundPredicate.java diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 245ec95a486..d536e101ab6 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -429,7 +429,7 @@ public class ConfigOptions { .defaultValue(0.10) .withDescription( "The maximum fraction of the total capacity of the volume containing the first available data directory allocated to historical partition lookup caches on a TabletServer. " - + "Up to ten table lookupers share this capacity. Historical lookup cache files are stored under that data directory; additional data volumes are not used. " + + "All table lookupers share this capacity, with eviction at file granularity. Historical lookup cache files are stored under that data directory; additional data volumes are not used. " + "The valid range is (0.0, 1.0]."); public static final ConfigOption @@ -438,7 +438,9 @@ public class ConfigOptions { .durationType() .defaultValue(Duration.ofHours(3)) .withDescription( - "The duration after which an idle historical partition table lookuper is removed from the cache."); + "The duration after which an idle historical partition table lookuper or an idle lookup file is removed from its cache. " + + "Lookuper and file access times are tracked independently. This setting replaces the Paimon table-level lookup.cache-file-retention option for historical lookups. " + + "Dynamic changes apply to both caches without replacing active lookupers."); public static final ConfigOption SERVER_DATA_DISK_WRITE_LIMIT_RATIO = key("server.data-disk.write-limit-ratio") diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java index 03ac4cf4e8d..7f2329bdd56 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java @@ -18,7 +18,7 @@ package org.apache.fluss.lake.lakestorage; import org.apache.fluss.annotation.PublicEvolving; -import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime.LookupRuntimeOptions; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupRuntimeOptions; import org.apache.fluss.lake.source.LakeSource; import org.apache.fluss.lake.writer.LakeTieringFactory; import org.apache.fluss.metadata.TablePath; @@ -53,13 +53,13 @@ public interface LakeStorage { LakeSource createLakeSource(TablePath tablePath); /** - * Creates a TabletServer-scoped runtime for lake table point lookup. + * Creates a TabletServer-scoped manager for lake table lookupers and their shared resources. * * @param ioTmpDir local directory shared by lookupers for temporary files * @param options initial runtime resource settings - * @return the lookup runtime + * @return the lookuper manager */ - default LakeTableLookupRuntime createLakeTableLookupRuntime( + default LakeTableLookuperManager createLakeTableLookuperManager( String ioTmpDir, LookupRuntimeOptions options) { throw new UnsupportedOperationException( "Point lookup is not supported for this lake storage."); diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookupRuntime.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuperManager.java similarity index 79% rename from fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookupRuntime.java rename to fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuperManager.java index 8656e1710bd..aa0f213cbb5 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookupRuntime.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuperManager.java @@ -22,12 +22,21 @@ import org.apache.fluss.config.TableConfig; import org.apache.fluss.metadata.TablePath; +import java.time.Duration; + import static org.apache.fluss.utils.Preconditions.checkArgument; import static org.apache.fluss.utils.Preconditions.checkNotNull; -/** TabletServer-scoped runtime managing shared resources and creating lake table lookupers. */ +/** + * Creates lake table lookupers and manages their shared resources within one TabletServer. + * + *

The caller must close all created lookupers after their requests have finished before closing + * this manager. Closing a lookuper must not release resources shared with other lookupers. + * + * @since 1.1 + */ @PublicEvolving -public interface LakeTableLookupRuntime extends AutoCloseable { +public interface LakeTableLookuperManager extends AutoCloseable { /** * Creates a table-level point lookuper for the specified lake table. @@ -50,6 +59,11 @@ public interface LakeTableLookupRuntime extends AutoCloseable { */ void reconfigure(LookupRuntimeOptions options); + /** Returns the cumulative number of lookup files evicted by the shared disk-space budget. */ + default long fileCacheCapacityEvictions() { + return 0L; + } + /** * Immutable snapshot of the format-independent resource settings for a lookup runtime. * @@ -59,6 +73,7 @@ public interface LakeTableLookupRuntime extends AutoCloseable { final class LookupRuntimeOptions { private final long localCacheMaxBytes; + private final Duration expireAfterAccess; /** * Creates runtime resource settings. @@ -66,16 +81,27 @@ final class LookupRuntimeOptions { * @param localCacheMaxBytes positive disk-space budget in bytes for local caches shared by * all lookupers in the runtime; implementations without local disk caches may ignore * this budget + * @param expireAfterAccess positive idle expiration for individual cached lookup files */ - public LookupRuntimeOptions(long localCacheMaxBytes) { + public LookupRuntimeOptions(long localCacheMaxBytes, Duration expireAfterAccess) { checkArgument(localCacheMaxBytes > 0, "localCacheMaxBytes must be greater than 0."); this.localCacheMaxBytes = localCacheMaxBytes; + this.expireAfterAccess = + checkNotNull(expireAfterAccess, "expireAfterAccess must not be null."); + checkArgument( + !expireAfterAccess.isNegative() && !expireAfterAccess.isZero(), + "expireAfterAccess must be greater than 0."); } /** Returns the runtime-wide disk-space budget for local caches, in bytes. */ public long localCacheMaxBytes() { return localCacheMaxBytes; } + + /** Returns the idle expiration applied independently to each cached lookup file. */ + public Duration expireAfterAccess() { + return expireAfterAccess; + } } /** Runtime context for creating a lake table lookuper. */ diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java index 42916ee9638..5cbfa194bea 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java @@ -20,7 +20,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.TableAlreadyExistException; import org.apache.fluss.exception.TableNotExistException; -import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime.LookupRuntimeOptions; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupRuntimeOptions; import org.apache.fluss.lake.source.LakeSource; import org.apache.fluss.lake.writer.LakeTieringFactory; import org.apache.fluss.metadata.TableChange; @@ -140,23 +140,23 @@ public LakeSource createLakeSource(TablePath tablePath) { } @Override - public LakeTableLookupRuntime createLakeTableLookupRuntime( + public LakeTableLookuperManager createLakeTableLookuperManager( String ioTmpDir, LookupRuntimeOptions options) { try (TemporaryClassLoaderContext ignored = TemporaryClassLoaderContext.of(loader)) { - return new ClassLoaderFixingLakeTableLookupRuntime( - inner.createLakeTableLookupRuntime(ioTmpDir, options), loader); + return new ClassLoaderFixingLakeTableLookuperManager( + inner.createLakeTableLookuperManager(ioTmpDir, options), loader); } } } - static class ClassLoaderFixingLakeTableLookupRuntime - implements LakeTableLookupRuntime, WrappingProxy { + static class ClassLoaderFixingLakeTableLookuperManager + implements LakeTableLookuperManager, WrappingProxy { - private final LakeTableLookupRuntime inner; + private final LakeTableLookuperManager inner; private final ClassLoader loader; - private ClassLoaderFixingLakeTableLookupRuntime( - LakeTableLookupRuntime inner, ClassLoader loader) { + private ClassLoaderFixingLakeTableLookuperManager( + LakeTableLookuperManager inner, ClassLoader loader) { this.inner = inner; this.loader = loader; } @@ -176,6 +176,13 @@ public void reconfigure(LookupRuntimeOptions options) { } } + @Override + public long fileCacheCapacityEvictions() { + try (TemporaryClassLoaderContext ignored = TemporaryClassLoaderContext.of(loader)) { + return inner.fileCacheCapacityEvictions(); + } + } + @Override public void close() throws Exception { try (TemporaryClassLoaderContext ignored = TemporaryClassLoaderContext.of(loader)) { @@ -184,7 +191,7 @@ public void close() throws Exception { } @Override - public LakeTableLookupRuntime getWrappedDelegate() { + public LakeTableLookuperManager getWrappedDelegate() { return inner; } } diff --git a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java index 0219f3fe79f..556d928e4e0 100644 --- a/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java +++ b/fluss-common/src/main/java/org/apache/fluss/metrics/MetricNames.java @@ -112,8 +112,8 @@ public class MetricNames { // for historical lookup cache public static final String HISTORICAL_LOOKUP_CACHE_DISK_SIZE = "lookupCacheDiskSize"; public static final String HISTORICAL_LOOKUP_CACHE_TABLE_COUNT = "lookupCacheTableCount"; - public static final String HISTORICAL_LOOKUP_CACHE_CAPACITY_EVICTIONS = - "lookupCacheCapacityEvictions"; + public static final String HISTORICAL_LOOKUP_CACHE_FILE_CAPACITY_EVICTIONS = + "lookupCacheFileCapacityEvictions"; // -------------------------------------------------------------------------------------------- // metrics for user diff --git a/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java b/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java index 7374a937015..c8e14218a20 100644 --- a/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java @@ -20,7 +20,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.TableAlreadyExistException; import org.apache.fluss.exception.TableNotExistException; -import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime.LookupRuntimeOptions; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupRuntimeOptions; import org.apache.fluss.lake.source.LakeSource; import org.apache.fluss.lake.writer.LakeTieringFactory; import org.apache.fluss.metadata.TableChange; @@ -30,6 +30,7 @@ import org.junit.jupiter.api.Test; +import java.time.Duration; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; @@ -43,6 +44,18 @@ class LakeStorageTest { private static final String TEST_LAKE_PLUGIN_FORMAT = "test-plugin"; + @Test + void testLookupRuntimeOptionsRejectInvalidResourceLimits() { + assertThatThrownBy(() -> new LookupRuntimeOptions(0L, Duration.ofHours(1))) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new LookupRuntimeOptions(1024L, Duration.ZERO)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new LookupRuntimeOptions(1024L, Duration.ofSeconds(-1))) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> new LookupRuntimeOptions(1024L, null)) + .isInstanceOf(NullPointerException.class); + } + @Test void testContextWithoutCurrentLakeTablePath() { assertThat(new TestingLakeCatalogContext().getCurrentLakeTablePath()).isNull(); @@ -98,19 +111,24 @@ void testWithPluginManager() throws Exception { .getWrappedDelegate()) .isInstanceOf(TestPaimonLakeCatalog.class); - LakeTableLookupRuntime lookupRuntime = - lakeStorage.createLakeTableLookupRuntime( - "lookup-dir", new LookupRuntimeOptions(1024L)); - assertThat(lookupRuntime) + LakeTableLookuperManager lookuperManager = + lakeStorage.createLakeTableLookuperManager( + "lookup-dir", new LookupRuntimeOptions(1024L, Duration.ofHours(3))); + assertThat(lookuperManager) .isInstanceOf( - PluginLakeStorageWrapper.ClassLoaderFixingLakeTableLookupRuntime.class); - TestLakeTableLookupRuntime innerLookupRuntime = - (TestLakeTableLookupRuntime) - ((PluginLakeStorageWrapper.ClassLoaderFixingLakeTableLookupRuntime) - lookupRuntime) + PluginLakeStorageWrapper.ClassLoaderFixingLakeTableLookuperManager.class); + TestLakeTableLookuperManager innerLookuperManager = + (TestLakeTableLookuperManager) + ((PluginLakeStorageWrapper.ClassLoaderFixingLakeTableLookuperManager) + lookuperManager) .getWrappedDelegate(); - lookupRuntime.close(); - assertThat(innerLookupRuntime.closed).isTrue(); + LookupRuntimeOptions updatedOptions = + new LookupRuntimeOptions(2048L, Duration.ofMinutes(30)); + lookuperManager.reconfigure(updatedOptions); + assertThat(innerLookuperManager.options).isSameAs(updatedOptions); + assertThat(lookuperManager.fileCacheCapacityEvictions()).isEqualTo(3L); + lookuperManager.close(); + assertThat(innerLookuperManager.closed).isTrue(); } private static class TestingPluginManager implements PluginManager { @@ -162,15 +180,16 @@ public LakeSource createLakeSource(TablePath tablePath) { } @Override - public LakeTableLookupRuntime createLakeTableLookupRuntime( + public LakeTableLookuperManager createLakeTableLookuperManager( String ioTmpDir, LookupRuntimeOptions options) { - return new TestLakeTableLookupRuntime(); + return new TestLakeTableLookuperManager(); } } - private static class TestLakeTableLookupRuntime implements LakeTableLookupRuntime { + private static class TestLakeTableLookuperManager implements LakeTableLookuperManager { private boolean closed; + private LookupRuntimeOptions options; @Override public LakeTableLookuper createLakeTableLookuper(TablePath tablePath, Context context) { @@ -178,7 +197,14 @@ public LakeTableLookuper createLakeTableLookuper(TablePath tablePath, Context co } @Override - public void reconfigure(LookupRuntimeOptions options) {} + public void reconfigure(LookupRuntimeOptions options) { + this.options = options; + } + + @Override + public long fileCacheCapacityEvictions() { + return 3L; + } @Override public void close() { diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java index bb853f3a45d..349996ebf74 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java @@ -19,9 +19,9 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.lake.lakestorage.LakeStorage; -import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime; -import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime.LookupRuntimeOptions; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupRuntimeOptions; import org.apache.fluss.lake.paimon.lookup.PaimonLakeTableLookuper; import org.apache.fluss.lake.paimon.lookup.PaimonScanBasedTableLookuper; import org.apache.fluss.lake.paimon.lookup.SharedLookupFileCache; @@ -36,7 +36,6 @@ import org.apache.fluss.metadata.TablePath; import org.apache.fluss.utils.IOUtils; -import org.apache.paimon.CoreOptions; import org.apache.paimon.disk.IOManager; import org.apache.paimon.options.MemorySize; @@ -67,22 +66,22 @@ public LakeSource createLakeSource(TablePath tablePath) { } @Override - public LakeTableLookupRuntime createLakeTableLookupRuntime( + public LakeTableLookuperManager createLakeTableLookuperManager( String ioTmpDir, LookupRuntimeOptions options) { - return new PaimonLakeTableLookupRuntime(ioTmpDir, options); + return new PaimonLakeTableLookuperManager(ioTmpDir, options); } - /** Paimon lookup runtime sharing one I/O manager across table lookupers. */ - private static final class PaimonLakeTableLookupRuntime implements LakeTableLookupRuntime { + /** Owns the shared I/O manager and file cache used by Paimon table lookupers. */ + private static final class PaimonLakeTableLookuperManager implements LakeTableLookuperManager { private final IOManager ioManager; private final SharedLookupFileCache lookupFileCache; - private PaimonLakeTableLookupRuntime(String ioTmpDir, LookupRuntimeOptions options) { + private PaimonLakeTableLookuperManager(String ioTmpDir, LookupRuntimeOptions options) { checkNotNull(options, "options must not be null."); this.ioManager = IOManager.create(checkNotNull(ioTmpDir, "ioTmpDir must not be null.")); this.lookupFileCache = new SharedLookupFileCache( - CoreOptions.LOOKUP_CACHE_FILE_RETENTION.defaultValue(), + options.expireAfterAccess(), new MemorySize(options.localCacheMaxBytes())); } @@ -108,6 +107,12 @@ public LakeTableLookuper createLakeTableLookuper(TablePath tablePath, Context co public void reconfigure(LookupRuntimeOptions options) { checkNotNull(options, "options must not be null."); lookupFileCache.updateMaxDiskSize(new MemorySize(options.localCacheMaxBytes())); + lookupFileCache.updateExpireAfterAccess(options.expireAfterAccess()); + } + + @Override + public long fileCacheCapacityEvictions() { + return lookupFileCache.capacityEvictions(); } @Override diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/NamespacedLookupFileCache.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/NamespacedLookupFileCache.java index 0eb408101ff..611d8813b97 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/NamespacedLookupFileCache.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/NamespacedLookupFileCache.java @@ -22,6 +22,7 @@ import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Policy; import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.stats.CacheStats; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.AbstractMap; @@ -47,18 +48,19 @@ final class NamespacedLookupFileCache implements Cache { } @Override - public @Nullable LookupFile getIfPresent(Object fileName) { + public @Nullable LookupFile getIfPresent(@Nonnull Object fileName) { return fileName instanceof String ? sharedCache.getIfPresent(key((String) fileName)) : null; } @Override - public LookupFile get( - String fileName, Function mappingFunction) { + public @Nullable LookupFile get( + @Nonnull String fileName, + @Nonnull Function mappingFunction) { return sharedCache.get(key(fileName), ignored -> mappingFunction.apply(fileName)); } @Override - public Map getAllPresent(Iterable fileNames) { + public @Nonnull Map getAllPresent(@Nonnull Iterable fileNames) { Map result = new LinkedHashMap<>(); for (Object fileName : fileNames) { LookupFile lookupFile = getIfPresent(fileName); @@ -70,24 +72,24 @@ public Map getAllPresent(Iterable fileNames) { } @Override - public void put(String fileName, LookupFile lookupFile) { + public void put(@Nonnull String fileName, @Nonnull LookupFile lookupFile) { sharedCache.put(key(fileName), lookupFile); } @Override - public void putAll(Map entries) { + public void putAll(@Nonnull Map entries) { entries.forEach(this::put); } @Override - public void invalidate(Object fileName) { + public void invalidate(@Nonnull Object fileName) { if (fileName instanceof String) { sharedCache.invalidate(key((String) fileName)); } } @Override - public void invalidateAll(Iterable fileNames) { + public void invalidateAll(@Nonnull Iterable fileNames) { for (Object fileName : fileNames) { invalidate(fileName); } @@ -95,8 +97,8 @@ public void invalidateAll(Iterable fileNames) { @Override public void invalidateAll() { - // ponytail: O(n) namespace scan; add a namespace index if cache cardinality makes close - // slow. + // This view owns only its namespace; never invalidate the entire shared cache. + // ponytail: O(n) scan of all cached files; add a namespace index if close becomes slow. Set keys = new HashSet<>(); for (SharedLookupFileCache.Key key : sharedCache.asMap().keySet()) { if (key.namespace.equals(namespace)) { @@ -114,12 +116,12 @@ public long estimatedSize() { } @Override - public CacheStats stats() { + public @Nonnull CacheStats stats() { return sharedCache.stats(); } @Override - public ConcurrentMap asMap() { + public @Nonnull ConcurrentMap asMap() { return new NamespacedMap(); } @@ -129,7 +131,7 @@ public void cleanUp() { } @Override - public Policy policy() { + public @Nonnull Policy policy() { throw new UnsupportedOperationException( "Policy access is not supported by the namespaced cache view."); } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLocalTableQuery.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLocalTableQuery.java index 5d2ba9e262b..1dd75fc1290 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLocalTableQuery.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLocalTableQuery.java @@ -66,8 +66,13 @@ import static org.apache.paimon.mergetree.LookupFile.localFilePrefix; /** - * Fluss-owned fork of Paimon 2.0.0's {@code LocalTableQuery}, using a shared namespaced lookup - * cache. + * Adapted from Paimon 2.0.0's LocalTableQuery. + * + *

Accepts a shared lookup-file cache and an instance namespace, passing that namespace's view to + * {@link LookupLevels}. This replaces the lazy, query-owned cache built from table retention and + * disk-size options. Closing a query only invalidates its namespace; the shared cache and I/O + * manager belong to the lake table lookuper manager. */ final class PaimonLocalTableQuery implements TableQuery { diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/SharedLookupFileCache.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/SharedLookupFileCache.java index 1e99d914c83..d5ac32a7d61 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/SharedLookupFileCache.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/SharedLookupFileCache.java @@ -18,12 +18,14 @@ package org.apache.fluss.lake.paimon.lookup; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.annotation.VisibleForTesting; import org.apache.paimon.mergetree.LookupFile; import org.apache.paimon.options.MemorySize; import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Caffeine; import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.RemovalCause; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Ticker; import javax.annotation.Nullable; @@ -31,6 +33,8 @@ import java.io.UncheckedIOException; import java.time.Duration; import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import static org.apache.fluss.utils.Preconditions.checkNotNull; import static org.apache.paimon.mergetree.LookupUtils.fileKibiBytes; @@ -40,9 +44,15 @@ public final class SharedLookupFileCache implements AutoCloseable { private final Cache cache; + private final AtomicLong capacityEvictions = new AtomicLong(); /** Creates a shared lookup-file cache. */ public SharedLookupFileCache(Duration fileRetention, MemorySize maxDiskSize) { + this(fileRetention, maxDiskSize, Ticker.systemTicker()); + } + + @VisibleForTesting + SharedLookupFileCache(Duration fileRetention, MemorySize maxDiskSize, Ticker ticker) { checkNotNull(fileRetention, "fileRetention must not be null."); checkNotNull(maxDiskSize, "maxDiskSize must not be null."); this.cache = @@ -52,7 +62,8 @@ public SharedLookupFileCache(Duration fileRetention, MemorySize maxDiskSize) { .weigher( (Key key, LookupFile lookupFile) -> Math.max(1, fileKibiBytes(lookupFile.localFile()))) - .removalListener(SharedLookupFileCache::removeLookupFile) + .ticker(checkNotNull(ticker, "ticker must not be null.")) + .removalListener(this::removeLookupFile) .executor(Runnable::run) .build(); } @@ -66,15 +77,31 @@ public void updateMaxDiskSize(MemorySize maxDiskSize) { cache.policy().eviction().get().setMaximum(Math.max(1L, maxDiskSize.getKibiBytes())); } + /** Updates idle expiration without replacing cached lookup files. */ + public void updateExpireAfterAccess(Duration expireAfterAccess) { + cache.policy() + .expireAfterAccess() + .get() + .setExpiresAfter(expireAfterAccess.toNanos(), TimeUnit.NANOSECONDS); + } + + /** Returns the number of files evicted to enforce the shared disk-space budget. */ + public long capacityEvictions() { + return capacityEvictions.get(); + } + @Override public void close() { cache.invalidateAll(); cache.cleanUp(); } - private static void removeLookupFile( + private void removeLookupFile( @Nullable Key key, @Nullable LookupFile lookupFile, RemovalCause cause) { if (lookupFile != null) { + if (cause == RemovalCause.SIZE) { + capacityEvictions.incrementAndGet(); + } try { lookupFile.close(cause); } catch (IOException e) { diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java index f89992db293..80af27310ac 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java @@ -25,9 +25,9 @@ import org.apache.fluss.exception.DiskWriteLockedException; import org.apache.fluss.exception.KvStorageException; import org.apache.fluss.exception.RetriableException; -import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime; -import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime.LookupRuntimeOptions; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupRuntimeOptions; import org.apache.fluss.lake.lakestorage.TestingLakeCatalogContext; import org.apache.fluss.lake.paimon.PaimonLakeCatalog; import org.apache.fluss.lake.paimon.PaimonLakeStorage; @@ -77,6 +77,7 @@ import java.io.IOException; import java.lang.reflect.Field; import java.nio.file.Files; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -119,7 +120,7 @@ class PaimonLakeTableLookuperTest { private Configuration paimonConfig; private PaimonLakeCatalog lakeCatalog; private Catalog paimonCatalog; - private LakeTableLookupRuntime lookupRuntime; + private LakeTableLookuperManager lookuperManager; @BeforeEach void setUp() { @@ -129,17 +130,18 @@ void setUp() { paimonCatalog = CatalogFactory.createCatalog( CatalogContext.create(Options.fromMap(paimonConfig.toMap()))); - lookupRuntime = + lookuperManager = new PaimonLakeStorage(paimonConfig) - .createLakeTableLookupRuntime( + .createLakeTableLookuperManager( tempWarehouseDir.getAbsolutePath(), - new LookupRuntimeOptions(LOOKUP_CACHE_MAX_DISK_BYTES)); + new LookupRuntimeOptions( + LOOKUP_CACHE_MAX_DISK_BYTES, Duration.ofHours(3))); } @AfterEach void tearDown() throws Exception { - if (lookupRuntime != null) { - lookupRuntime.close(); + if (lookuperManager != null) { + lookuperManager.close(); } if (paimonCatalog != null) { paimonCatalog.close(); @@ -486,29 +488,30 @@ void testSharesIOManagerAndDeletesOnlyClosedLookuperFiles() throws Exception { 0, Collections.singletonList(paimonRow(2, "20240101", "Bob")))); File lookupDir = new File(tempWarehouseDir, "shared-lookup-cache"); - LakeTableLookupRuntime.Context firstLookuperContext = - new LakeTableLookupRuntime.Context( + LakeTableLookuperManager.Context firstLookuperContext = + new LakeTableLookuperManager.Context( paimonConfig, "first-table", tableConfig(KvFormat.COMPACTED, 1, LakeLookupMode.SST), NO_OP_DISK_WRITE_GUARD); - LakeTableLookupRuntime.Context secondLookuperContext = - new LakeTableLookupRuntime.Context( + LakeTableLookuperManager.Context secondLookuperContext = + new LakeTableLookuperManager.Context( paimonConfig, "second-table", tableConfig(KvFormat.COMPACTED, 1, LakeLookupMode.SST), NO_OP_DISK_WRITE_GUARD); - LakeTableLookupRuntime sharedLookupRuntime = + LakeTableLookuperManager sharedLookuperManager = new PaimonLakeStorage(paimonConfig) - .createLakeTableLookupRuntime( + .createLakeTableLookuperManager( lookupDir.getAbsolutePath(), - new LookupRuntimeOptions(LOOKUP_CACHE_MAX_DISK_BYTES)); + new LookupRuntimeOptions( + LOOKUP_CACHE_MAX_DISK_BYTES, Duration.ofHours(3))); try { try (LakeTableLookuper firstLookuper = - sharedLookupRuntime.createLakeTableLookuper( + sharedLookuperManager.createLakeTableLookuper( firstTablePath, firstLookuperContext); LakeTableLookuper secondLookuper = - sharedLookupRuntime.createLakeTableLookuper( + sharedLookuperManager.createLakeTableLookuper( secondTablePath, secondLookuperContext)) { assertThat( firstLookuper.lookup( @@ -540,7 +543,7 @@ void testSharesIOManagerAndDeletesOnlyClosedLookuperFiles() throws Exception { assertThat(regularFiles(lookupDir)).isEmpty(); assertThat(lookupDir.listFiles(File::isDirectory)).hasSize(1); } finally { - sharedLookupRuntime.close(); + sharedLookuperManager.close(); } assertThat(lookupDir.listFiles(File::isDirectory)).isEmpty(); } @@ -1156,9 +1159,9 @@ private static TableConfig tableConfig( private LakeTableLookuper createLookuper( TablePath tablePath, TableConfig tableConfig, Runnable diskWriteGuard) { - return lookupRuntime.createLakeTableLookuper( + return lookuperManager.createLakeTableLookuper( tablePath, - new LakeTableLookupRuntime.Context( + new LakeTableLookuperManager.Context( paimonConfig, tablePath.toString(), tableConfig, diskWriteGuard)); } diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/SharedLookupFileCacheTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/SharedLookupFileCacheTest.java index fa497dd3353..f38d2b4d7b4 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/SharedLookupFileCacheTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/SharedLookupFileCacheTest.java @@ -21,6 +21,7 @@ import org.apache.paimon.mergetree.LookupFile; import org.apache.paimon.options.MemorySize; import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache; +import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.RemovalCause; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -29,6 +30,13 @@ import java.io.RandomAccessFile; import java.time.Duration; import java.util.Arrays; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import static org.assertj.core.api.Assertions.assertThat; @@ -62,13 +70,157 @@ void testNamespaceIsolationAndGlobalLimit() throws Exception { sharedCache.updateMaxDiskSize(MemorySize.ofKibiBytes(1)); secondNamespace.put("third-file", lookupFile(thirdFile)); assertThat(Arrays.asList(secondFile, thirdFile).stream().filter(File::exists).count()) - .isLessThanOrEqualTo(1L); + .isEqualTo(1L); + assertThat(sharedCache.capacityEvictions()).isOne(); } assertThat(secondFile).doesNotExist(); assertThat(thirdFile).doesNotExist(); } + @Test + void testGlobalBudgetAcrossNamespaces() throws Exception { + File firstFile = lookupFile("first.lookup"); + File secondFile = lookupFile("second.lookup"); + try (SharedLookupFileCache sharedCache = + new SharedLookupFileCache(Duration.ofHours(1), MemorySize.ofKibiBytes(2))) { + Cache first = sharedCache.namespaced("first"); + Cache second = sharedCache.namespaced("second"); + first.put("file", lookupFile(firstFile)); + second.put("file", lookupFile(secondFile)); + first.cleanUp(); + assertThat(first.estimatedSize() + second.estimatedSize()).isEqualTo(2L); + + sharedCache.updateMaxDiskSize(MemorySize.ofKibiBytes(1)); + first.cleanUp(); + assertThat(first.estimatedSize() + second.estimatedSize()).isOne(); + assertThat(Arrays.asList(firstFile, secondFile)).filteredOn(File::exists).hasSize(1); + assertThat(sharedCache.capacityEvictions()).isOne(); + sharedCache.close(); + assertThat(firstFile).doesNotExist(); + assertThat(secondFile).doesNotExist(); + assertThat(sharedCache.capacityEvictions()).isOne(); + } + } + + @Test + void testDynamicExpirationAndIndependentFileAccess() throws Exception { + AtomicLong time = new AtomicLong(); + File firstFile = lookupFile("first.lookup"); + File secondFile = lookupFile("second.lookup"); + try (SharedLookupFileCache sharedCache = + new SharedLookupFileCache( + Duration.ofHours(3), MemorySize.ofKibiBytes(2), time::get)) { + Cache cache = sharedCache.namespaced("table"); + cache.put("first", lookupFile(firstFile)); + cache.put("second", lookupFile(secondFile)); + time.set(Duration.ofMinutes(20).toNanos()); + assertThat(cache.getIfPresent("second")).isNotNull(); + sharedCache.updateExpireAfterAccess(Duration.ofMinutes(30)); + time.set(Duration.ofMinutes(31).toNanos()); + cache.cleanUp(); + assertThat(firstFile).doesNotExist(); + assertThat(secondFile).exists(); + assertThat(sharedCache.capacityEvictions()).isZero(); + + sharedCache.updateExpireAfterAccess(Duration.ofHours(2)); + time.set(Duration.ofHours(1).toNanos()); + cache.cleanUp(); + assertThat(secondFile).exists(); + time.set(Duration.ofHours(3).toNanos()); + cache.cleanUp(); + assertThat(secondFile).doesNotExist(); + assertThat(sharedCache.capacityEvictions()).isZero(); + } + } + + @Test + void testCapacityEvictionWaitsForActiveFileRead() throws Exception { + CountDownLatch readersStarted = new CountDownLatch(2); + CountDownLatch releaseReaders = new CountDownLatch(1); + CountDownLatch evictionStarted = new CountDownLatch(1); + AtomicInteger readersClosed = new AtomicInteger(); + File firstFile = lookupFile("first.lookup"); + File secondFile = lookupFile("second.lookup"); + ExecutorService executor = Executors.newFixedThreadPool(3); + try (SharedLookupFileCache sharedCache = + new SharedLookupFileCache(Duration.ofHours(1), MemorySize.ofKibiBytes(2))) { + LookupFile first = + blockingLookupFile( + firstFile, + readersStarted, + releaseReaders, + evictionStarted, + readersClosed); + LookupFile second = + blockingLookupFile( + secondFile, + readersStarted, + releaseReaders, + evictionStarted, + readersClosed); + sharedCache.namespaced("first").put("file", first); + sharedCache.namespaced("second").put("file", second); + Future firstRead = executor.submit(() -> first.get(new byte[] {1})); + Future secondRead = executor.submit(() -> second.get(new byte[] {2})); + try { + assertThat(readersStarted.await(30, TimeUnit.SECONDS)).isTrue(); + Future eviction = + executor.submit( + () -> sharedCache.updateMaxDiskSize(MemorySize.ofKibiBytes(1))); + assertThat(evictionStarted.await(30, TimeUnit.SECONDS)).isTrue(); + assertThat(readersClosed.get()).isZero(); + assertThat(firstFile).exists(); + assertThat(secondFile).exists(); + releaseReaders.countDown(); + assertThat(firstRead.get(30, TimeUnit.SECONDS)).containsExactly((byte) 1); + assertThat(secondRead.get(30, TimeUnit.SECONDS)).containsExactly((byte) 2); + eviction.get(30, TimeUnit.SECONDS); + assertThat(readersClosed.get()).isOne(); + assertThat(sharedCache.capacityEvictions()).isOne(); + } finally { + releaseReaders.countDown(); + } + } finally { + releaseReaders.countDown(); + executor.shutdownNow(); + } + } + + private static LookupFile blockingLookupFile( + File file, + CountDownLatch started, + CountDownLatch release, + CountDownLatch evictionStarted, + AtomicInteger closed) { + LookupStoreReader reader = + new LookupStoreReader() { + @Override + public byte[] lookup(byte[] key) throws IOException { + started.countDown(); + try { + release.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException(e); + } + return key; + } + + @Override + public void close() { + closed.incrementAndGet(); + } + }; + return new LookupFile(file, 1, 0L, "v1", reader, () -> {}) { + @Override + public void close(RemovalCause cause) throws IOException { + evictionStarted.countDown(); + super.close(cause); + } + }; + } + private File lookupFile(String name) throws IOException { File file = new File(tempDir, name); try (RandomAccessFile randomAccessFile = new RandomAccessFile(file, "rw")) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index 292faf37e7a..41ce69fce7a 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -479,9 +479,9 @@ private void registerMetrics() { historicalMetrics.gauge( MetricNames.HISTORICAL_LOOKUP_CACHE_TABLE_COUNT, historicalPartitionManager::cachedTableCount); - historicalMetrics.counter( - MetricNames.HISTORICAL_LOOKUP_CACHE_CAPACITY_EVICTIONS, - historicalPartitionManager.capacityEvictions()); + historicalMetrics.gauge( + MetricNames.HISTORICAL_LOOKUP_CACHE_FILE_CAPACITY_EVICTIONS, + historicalPartitionManager::fileCacheCapacityEvictions); serverMetricGroup.gauge( MetricNames.REPLICA_LEADER_COUNT, diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java index f1c469de1a5..13a0bacf2c0 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java @@ -26,9 +26,9 @@ import org.apache.fluss.lake.lakestorage.LakeStorage; import org.apache.fluss.lake.lakestorage.LakeStoragePlugin; import org.apache.fluss.lake.lakestorage.LakeStoragePluginSetUp; -import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime; -import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime.LookupRuntimeOptions; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupRuntimeOptions; import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.LakeLookupMode; import org.apache.fluss.metadata.ResolvedPartitionSpec; @@ -36,8 +36,6 @@ import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableInfo; import org.apache.fluss.metadata.TablePath; -import org.apache.fluss.metrics.Counter; -import org.apache.fluss.metrics.ThreadSafeSimpleCounter; import org.apache.fluss.plugin.PluginManager; import org.apache.fluss.server.entity.LookupDataForBucket; import org.apache.fluss.server.storage.LocalDiskManager; @@ -68,7 +66,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Stream; import static org.apache.fluss.server.utils.LakeStorageUtils.extractLakeProperties; @@ -87,21 +87,21 @@ * closed after its last lookup releases it. A new required lake snapshot refreshes the cached * lookuper in place. * - *

One lake-format-specific lookup runtime is initialized with this manager, or when Paimon is - * configured dynamically, and shared by all table lookupers. The runtime owns TabletServer-scoped - * resources such as Paimon's I/O manager. + *

One lake-format-specific lookuper manager is initialized after startup directory cleanup, or + * when Paimon is configured dynamically, and shared by all table lookupers. It owns + * TabletServer-scoped resources such as Paimon's I/O manager. * - *

Up to ten table lookupers are cached. Their lookup files share one server-level disk budget, - * and Caffeine evicts lookupers when the table limit is exceeded. + *

Lookup files share one server-level disk budget. Files are evicted independently of the number + * of cached table lookupers. * *

Historical lookup cache I/O participates in TabletServer disk write protection. Existing cache * hits remain available when the data disk is write-locked, while lookups that need to download new * cache files are rejected until the disk recovers. * - *

A lookuper is closed when replaced, explicitly invalidated by a replica lifecycle event, - * evicted when the table limit is exceeded, the manager shuts down, or after the configured idle - * expiration. Caffeine expiration is scheduled on the shared TabletServer scheduler, allowing idle - * resources to be released even if no subsequent lookup accesses the cache. + *

A lookuper is closed when replaced, explicitly invalidated by a replica lifecycle event, the + * manager shuts down, or after the configured idle expiration. Caffeine expiration is scheduled on + * the shared TabletServer scheduler, allowing idle resources to be released even if no subsequent + * lookup accesses the cache. */ class HistoricalLakeLookupManager implements AutoCloseable { @@ -112,13 +112,16 @@ class HistoricalLakeLookupManager implements AutoCloseable { private static final String LOOKUP_CACHE_DISK_SIZE_TASK_NAME = "historical-lookup-cache-disk-size"; private static final Duration LOOKUP_CACHE_DISK_SIZE_CHECK_INTERVAL = Duration.ofMinutes(3); - private static final int MAX_CACHED_TABLES = 10; private volatile Configuration conf; private volatile long lakeConfigVersion; private final @Nullable PluginManager pluginManager; - private volatile @Nullable LakeTableLookupRuntime lookupRuntime; - private final Counter capacityEvictions; + private volatile @Nullable LakeTableLookuperManager lookuperManager; + // Acquisitions may run concurrently. Closing first prevents any new cache entries, then + // invalidates existing ones without holding this lock. In-flight lookups drain independently. + private final ReentrantReadWriteLock lifecycleLock = new ReentrantReadWriteLock(); + // Includes invalidated lookupers until their close operation has actually finished. + private final AtomicInteger openLookupers = new AtomicInteger(); private final AtomicLong lookuperIdSequence; private final Cache lakeTableLookupers; private final ConcurrentMap requiredLakeSnapshotIds = new ConcurrentHashMap<>(); @@ -133,6 +136,7 @@ class HistoricalLakeLookupManager implements AutoCloseable { private volatile long lookupCacheDiskSize; private volatile boolean started; + private volatile boolean closed; /** Creates a historical lake lookup manager. */ HistoricalLakeLookupManager( @@ -175,10 +179,8 @@ class HistoricalLakeLookupManager implements AutoCloseable { conf.get( ConfigOptions .SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO)); - this.capacityEvictions = new ThreadSafeSimpleCounter(); this.lakeTableLookupers = Caffeine.newBuilder() - .maximumSize(MAX_CACHED_TABLES) .expireAfterAccess( conf.get( ConfigOptions @@ -189,7 +191,6 @@ class HistoricalLakeLookupManager implements AutoCloseable { .removalListener(this::onLookuperRemoved) .build(); this.lookuperIdSequence = new AtomicLong(); - this.lookupRuntime = createLookupRuntime(conf); } private static com.github.benmanes.caffeine.cache.Scheduler createCacheScheduler( @@ -212,6 +213,7 @@ private static com.github.benmanes.caffeine.cache.Scheduler createCacheScheduler */ synchronized void startup(Scheduler scheduler) { checkNotNull(scheduler, "scheduler must not be null."); + checkState(!closed, "Historical lake lookup manager is closed."); if (started) { return; } @@ -231,6 +233,7 @@ synchronized void startup(Scheduler scheduler) { + historicalLookupCacheRootDir, e); } + lookuperManager = createLookuperManager(conf); scheduler.schedule( LOOKUP_CACHE_DISK_SIZE_TASK_NAME, this::updateLookupCacheDiskSize, @@ -271,15 +274,33 @@ List lookup( @Override public void close() { + lifecycleLock.writeLock().lock(); + try { + synchronized (this) { + if (closed) { + return; + } + closed = true; + } + } finally { + lifecycleLock.writeLock().unlock(); + } lakeTableLookupers.invalidateAll(); lakeTableLookupers.cleanUp(); requiredLakeSnapshotIds.clear(); - LakeTableLookupRuntime runtime; + closeSharedResourcesIfUnused(); + } + + private void closeSharedResourcesIfUnused() { + LakeTableLookuperManager manager; synchronized (this) { - runtime = lookupRuntime; - lookupRuntime = null; + if (!closed || openLookupers.get() != 0) { + return; + } + manager = lookuperManager; + lookuperManager = null; } - IOUtils.closeQuietly(runtime, "historical lake lookup runtime"); + IOUtils.closeQuietly(manager, "historical lake lookuper manager"); } /** Invalidates the cached lake lookuper for the given table. */ @@ -298,14 +319,15 @@ int cachedTableCount() { return lakeTableLookupers.asMap().size(); } - /** Returns the counter for table lookuper evictions caused by the cached table limit. */ - Counter capacityEvictions() { - return capacityEvictions; + /** Returns the cumulative number of files evicted by the shared disk-space budget. */ + long fileCacheCapacityEvictions() { + LakeTableLookuperManager manager = lookuperManager; + return manager == null ? 0L : manager.fileCacheCapacityEvictions(); } @VisibleForTesting - boolean hasLookupRuntime() { - return lookupRuntime != null; + boolean hasLookuperManager() { + return lookuperManager != null; } @VisibleForTesting @@ -324,6 +346,7 @@ void reconfigure(Configuration newConf) { ConfigOptions .SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS); synchronized (this) { + checkState(!closed, "Historical lake lookup manager is closed."); long newMaxDiskBytes = cacheBytes( newConf.get( @@ -338,11 +361,12 @@ void reconfigure(Configuration newConf) { conf.get( ConfigOptions .SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS)); - if (lakeConfigChanged && lookupRuntime == null) { - lookupRuntime = createLookupRuntime(newConf); + if (started && lakeConfigChanged && lookuperManager == null) { + lookuperManager = createLookuperManager(newConf); } - if (cacheLimitChanged && lookupRuntime != null) { - lookupRuntime.reconfigure(new LookupRuntimeOptions(newMaxDiskBytes)); + if ((cacheLimitChanged || expirationChanged) && lookuperManager != null) { + lookuperManager.reconfigure( + new LookupRuntimeOptions(newMaxDiskBytes, newExpiration)); } // Publish the configuration before its version. A lookup that observes the new version // must also observe the matching configuration snapshot. @@ -359,10 +383,9 @@ void reconfigure(Configuration newConf) { .setExpiresAfter(newExpiration.toMillis(), TimeUnit.MILLISECONDS); } if (lakeConfigChanged) { - // Do not invalidate while holding this monitor: lookuper creation holds a cache key - // lock before preparing the lookup directory under the same monitor. Invalidation - // closes inactive lookupers immediately and active lookupers after their last lookup - // releases them. + // Do not invalidate while holding this monitor: closing a lookuper may need it to + // release shared resources. Inactive lookupers close immediately; active lookupers + // close after their last lookup releases them. lakeTableLookupers.invalidateAll(); lakeTableLookupers.cleanUp(); } @@ -373,14 +396,6 @@ private void onLookuperRemoved( if (cachedLookuper == null) { return; } - if (cause == RemovalCause.SIZE) { - capacityEvictions.inc(); - LOG.info( - "Evicted historical lookup cache for table {} (table ID {}) because the cache retains at most {} tables.", - cachedLookuper.tablePath, - cachedLookuper.tableId, - MAX_CACHED_TABLES); - } cachedLookuper.invalidate(); } @@ -432,21 +447,23 @@ LakeTableLookuper createLakeTableLookuper( "Historical lookup requires cluster lake storage properties to be configured."); } - LakeTableLookupRuntime runtime = lookupRuntime; - if (runtime == null) { + LakeTableLookuperManager manager = lookuperManager; + if (manager == null) { throw new LakeStorageNotConfiguredException( - "Historical lake lookup runtime has not been initialized."); + "Historical lake lookuper manager has not been initialized."); } - return runtime.createLakeTableLookuper( + return manager.createLakeTableLookuper( tablePath, - new LakeTableLookupRuntime.Context( + new LakeTableLookuperManager.Context( Configuration.fromMap(lakeProperties), cacheNamespace, tableConfig, diskWriteGuard)); } - private @Nullable LakeTableLookupRuntime createLookupRuntime(Configuration configuration) { + @VisibleForTesting + @Nullable + LakeTableLookuperManager createLookuperManager(Configuration configuration) { DataLakeFormat dataLakeFormat = configuration.get(ConfigOptions.DATALAKE_FORMAT); Map lakeProperties = extractLakeProperties(configuration); if (dataLakeFormat != DataLakeFormat.PAIMON || lakeProperties == null) { @@ -456,9 +473,13 @@ LakeTableLookuper createLakeTableLookuper( LakeStoragePluginSetUp.fromDataLakeFormat(dataLakeFormat.toString(), pluginManager); LakeStorage lakeStorage = lakeStoragePlugin.createLakeStorage(Configuration.fromMap(lakeProperties)); - return lakeStorage.createLakeTableLookupRuntime( + return lakeStorage.createLakeTableLookuperManager( historicalLookupCacheRootDir.getAbsolutePath(), - new LookupRuntimeOptions(lookupCacheMaxDiskBytes)); + new LookupRuntimeOptions( + lookupCacheMaxDiskBytes, + configuration.get( + ConfigOptions + .SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS))); } private static boolean hasLakeConfigChanged(Configuration currentConf, Configuration newConf) { @@ -477,7 +498,10 @@ private long cacheBytes(double ratio) { private static String cacheNamespace( long tableId, int schemaId, long lakeConfigVersion, long lookuperId) { - return tableId + "-" + schemaId + "-" + lakeConfigVersion + "-" + lookuperId; + // Each creation gets a new lookuperId, even for the same table/schema/configuration. + // An invalidated instance can still serve in-flight requests; its eventual close must + // not invalidate files belonging to its replacement. + return String.format("%d-%d-%d-%d", tableId, schemaId, lakeConfigVersion, lookuperId); } /** Returns the most recently sampled historical lookup cache footprint, in bytes. */ @@ -512,59 +536,67 @@ private static long fileSize(Path path) { } } - private static void closeLookuper(CachedLakeTableLookuper cachedLookuper) { - IOUtils.closeQuietly(cachedLookuper.lookuper, "historical lake table lookuper"); + private void closeLookuper(CachedLakeTableLookuper cachedLookuper) { + try { + IOUtils.closeQuietly(cachedLookuper.lookuper, "historical lake table lookuper"); + } finally { + openLookupers.decrementAndGet(); + closeSharedResourcesIfUnused(); + } } private CachedLakeTableLookuper acquireLookuper(LookupContext context, TableInfo tableInfo) { - long currentLakeConfigVersion = lakeConfigVersion; - Configuration currentConf = conf; - LakeLookupMode lookupMode = tableInfo.getTableConfig().getHistoricalLookupMode(); - return lakeTableLookupers - .asMap() - .compute( - context.tableId, - (ignored, currentLookuper) -> { - // Read inside the per-table atomic update to avoid refreshing back to a - // snapshot captured while waiting for another lookup to finish updating - // this lookuper. - Long requiredLakeSnapshotId = - requiredLakeSnapshotIds.get(context.tableId); - CachedLakeTableLookuper selectedLookuper = currentLookuper; - // Create the lookuper lazily, and recreate it after schema, - // lookup mode, or lake configuration changes so it reloads lake - // table/query state and uses the current settings. - if (selectedLookuper == null - || selectedLookuper.schemaId != context.schemaId - || selectedLookuper.lookupMode != lookupMode - || selectedLookuper.lakeConfigVersion - != currentLakeConfigVersion) { - LakeTableLookuper lookuper = - createLakeTableLookuper( - context.tablePath, - tableInfo.getTableConfig(), - cacheNamespace( - context.tableId, - context.schemaId, - currentLakeConfigVersion, - lookuperIdSequence.getAndIncrement()), - currentConf); - selectedLookuper = - new CachedLakeTableLookuper( - context.tableId, - context.tablePath, - context.schemaId, - lookupMode, - currentLakeConfigVersion, - requiredLakeSnapshotId, - lookuper); - } - // Pin the lookuper before leaving the atomic cache update. - // Eviction or invalidation can then defer closing it until - // this lookup releases it. - selectedLookuper.acquire(requiredLakeSnapshotId); - return selectedLookuper; - }); + lifecycleLock.readLock().lock(); + try { + checkState(!closed, "Historical lake lookup manager is closed."); + long currentLakeConfigVersion = lakeConfigVersion; + Configuration currentConf = conf; + LakeLookupMode lookupMode = tableInfo.getTableConfig().getHistoricalLookupMode(); + return lakeTableLookupers + .asMap() + .compute( + context.tableId, + (ignored, currentLookuper) -> { + // Read within the atomic update so a waiting lookup cannot + // refresh back to a previously captured snapshot. + Long requiredLakeSnapshotId = + requiredLakeSnapshotIds.get(context.tableId); + CachedLakeTableLookuper selectedLookuper = currentLookuper; + // Create lazily; schema, lookup mode, or lake configuration + // changes require a new table/query instance with the current + // settings. + if (selectedLookuper == null + || selectedLookuper.schemaId != context.schemaId + || selectedLookuper.lookupMode != lookupMode + || selectedLookuper.lakeConfigVersion + != currentLakeConfigVersion) { + LakeTableLookuper lookuper = + createLakeTableLookuper( + context.tablePath, + tableInfo.getTableConfig(), + cacheNamespace( + context.tableId, + context.schemaId, + currentLakeConfigVersion, + lookuperIdSequence.getAndIncrement()), + currentConf); + selectedLookuper = + new CachedLakeTableLookuper( + context.schemaId, + lookupMode, + currentLakeConfigVersion, + requiredLakeSnapshotId, + lookuper); + } + // Pin the lookuper before leaving the atomic cache update. + // Eviction or invalidation can then defer closing it until + // this lookup releases it. + selectedLookuper.acquire(requiredLakeSnapshotId); + return selectedLookuper; + }); + } finally { + lifecycleLock.readLock().unlock(); + } } private static final class LookupContext { @@ -585,9 +617,7 @@ private LookupContext( } } - private static final class CachedLakeTableLookuper { - private final long tableId; - private final TablePath tablePath; + private final class CachedLakeTableLookuper { private final int schemaId; private final LakeLookupMode lookupMode; private final long lakeConfigVersion; @@ -600,20 +630,17 @@ private static final class CachedLakeTableLookuper { private boolean closed; private CachedLakeTableLookuper( - long tableId, - TablePath tablePath, int schemaId, LakeLookupMode lookupMode, long lakeConfigVersion, @Nullable Long lakeSnapshotId, LakeTableLookuper lookuper) { - this.tableId = tableId; - this.tablePath = tablePath; this.schemaId = schemaId; this.lookupMode = lookupMode; this.lakeConfigVersion = lakeConfigVersion; this.lakeSnapshotId = lakeSnapshotId; this.lookuper = lookuper; + openLookupers.incrementAndGet(); } private synchronized void acquire(@Nullable Long requiredLakeSnapshotId) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java index cf5302e685f..4b1810b3c2a 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalPartitionManager.java @@ -26,7 +26,6 @@ import org.apache.fluss.metadata.ResolvedPartitionSpec; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TableInfo; -import org.apache.fluss.metrics.Counter; import org.apache.fluss.plugin.PluginManager; import org.apache.fluss.row.encode.KvValueLayout; import org.apache.fluss.rpc.entity.LookupResultForBucket; @@ -211,9 +210,9 @@ public int cachedTableCount() { return lakeLookupManager.cachedTableCount(); } - /** Returns the counter for lookuper evictions caused by the table cache capacity. */ - public Counter capacityEvictions() { - return lakeLookupManager.capacityEvictions(); + /** Returns the cumulative number of lookup files evicted by the shared disk-space budget. */ + public long fileCacheCapacityEvictions() { + return lakeLookupManager.fileCacheCapacityEvictions(); } @VisibleForTesting diff --git a/fluss-server/src/test/java/org/apache/fluss/server/lakehouse/TestingPaimonStoragePlugin.java b/fluss-server/src/test/java/org/apache/fluss/server/lakehouse/TestingPaimonStoragePlugin.java index 9ce94b72cc8..5eb3d982864 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/lakehouse/TestingPaimonStoragePlugin.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/lakehouse/TestingPaimonStoragePlugin.java @@ -28,9 +28,9 @@ import org.apache.fluss.lake.lakestorage.LakeCatalog; import org.apache.fluss.lake.lakestorage.LakeStorage; import org.apache.fluss.lake.lakestorage.LakeStoragePlugin; -import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime; -import org.apache.fluss.lake.lakestorage.LakeTableLookupRuntime.LookupRuntimeOptions; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupRuntimeOptions; import org.apache.fluss.lake.serializer.SimpleVersionedSerializer; import org.apache.fluss.lake.source.LakeSource; import org.apache.fluss.lake.writer.LakeTieringFactory; @@ -91,9 +91,9 @@ public LakeSource createLakeSource(TablePath tablePath) { } @Override - public LakeTableLookupRuntime createLakeTableLookupRuntime( + public LakeTableLookuperManager createLakeTableLookuperManager( String ioTmpDir, LookupRuntimeOptions options) { - return new LakeTableLookupRuntime() { + return new LakeTableLookuperManager() { @Override public LakeTableLookuper createLakeTableLookuper( TablePath tablePath, Context context) { diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java index beecbe57661..507786f2f4e 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java @@ -22,6 +22,8 @@ import org.apache.fluss.config.MemorySize; import org.apache.fluss.config.TableConfig; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupRuntimeOptions; import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.KvFormat; import org.apache.fluss.metadata.LakeLookupMode; @@ -40,6 +42,7 @@ import com.github.benmanes.caffeine.cache.Ticker; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentCaptor; import java.io.File; import java.io.RandomAccessFile; @@ -47,14 +50,25 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.FutureTask; import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import static org.apache.fluss.record.TestData.PARTITION_TABLE_ID; import static org.apache.fluss.record.TestData.PARTITION_TABLE_INFO; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** Tests for {@link HistoricalLakeLookupManager}. */ class HistoricalLakeLookupManagerTest { @@ -78,7 +92,9 @@ void testCleansAndCreatesLookupCacheDirectoryOnStartup() throws Exception { TestingHistoricalLakeLookupManager manager = new TestingHistoricalLakeLookupManager(conf()); assertThat(staleLookupFile).exists(); + assertThat(manager.hasLookuperManager()).isFalse(); manager.startup(NO_OP_SCHEDULER); + assertThat(manager.hasLookuperManager()).isTrue(); assertThat(staleLookupFile).doesNotExist(); assertThat(serverLookupDir).isDirectory(); lookup(manager, PARTITION_TABLE_INFO); @@ -317,7 +333,7 @@ void testDynamicallyUpdatesExpirationAndExpiresIdleLookuper() throws Exception { } @Test - void testEvictsLookuperWhenCachedTableLimitIsExceeded() throws Exception { + void testRetainsMoreThanTenLookupersWithinSharedBudget() throws Exception { Configuration conf = conf(); conf.set(ConfigOptions.SERVER_HISTORICAL_PARTITION_LOOKUP_CACHE_MAX_DISK_RATIO, 0.20); TestingHistoricalLakeLookupManager manager = @@ -330,11 +346,10 @@ void testEvictsLookuperWhenCachedTableLimitIsExceeded() throws Exception { } assertThat(manager.createdLookupers).hasSize(11); - assertThat(manager.createdLookupers).filteredOn(lookuper -> lookuper.closed).hasSize(1); + assertThat(manager.createdLookupers).noneMatch(lookuper -> lookuper.closed); assertThat(manager.createdCacheNamespaces).doesNotHaveDuplicates(); assertThat(manager.lookupCacheMaxDiskBytes()).isEqualTo(20L); - assertThat(manager.cachedTableCount()).isEqualTo(10); - assertThat(manager.capacityEvictions().getCount()).isEqualTo(1); + assertThat(manager.cachedTableCount()).isEqualTo(11); } @Test @@ -360,6 +375,89 @@ void testUpdatesSharedCacheLimitWithoutReplacingLookuper() throws Exception { assertThat(manager.lookupCacheMaxDiskBytes()).isEqualTo(20L); assertThat(lookuper.closed).isFalse(); assertThat(manager.cachedTableCount()).isOne(); + ArgumentCaptor options = + ArgumentCaptor.forClass(LookupRuntimeOptions.class); + verify(manager.sharedManager).reconfigure(options.capture()); + assertThat(options.getValue().localCacheMaxBytes()).isEqualTo(20L); + assertThat(options.getValue().expireAfterAccess()).isEqualTo(Duration.ofHours(3)); + } + + @Test + void testUpdatesFileCacheExpirationWithoutReplacingLookuper() throws Exception { + TestingHistoricalLakeLookupManager manager = createTestingManager(); + lookup(manager, PARTITION_TABLE_INFO); + manager.reconfigure(confWithExpiration(Duration.ofMinutes(30))); + + ArgumentCaptor options = + ArgumentCaptor.forClass(LookupRuntimeOptions.class); + verify(manager.sharedManager).reconfigure(options.capture()); + assertThat(options.getValue().expireAfterAccess()).isEqualTo(Duration.ofMinutes(30)); + assertThat(manager.createdLookupers).hasSize(1).noneMatch(lookuper -> lookuper.closed); + when(manager.sharedManager.fileCacheCapacityEvictions()).thenReturn(7L); + assertThat(manager.fileCacheCapacityEvictions()).isEqualTo(7L); + } + + @Test + void testClosesSharedResourcesAfterInvalidatedLookuperFinishes() throws Exception { + TestingHistoricalLakeLookupManager manager = createTestingManager(); + CountDownLatch lookupStarted = new CountDownLatch(1); + CountDownLatch releaseLookup = new CountDownLatch(1); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future lookup = + executor.submit( + () -> { + lookup( + manager, + PARTITION_TABLE_INFO, + PARTITION_TABLE_INFO.getSchemaInfo(), + (nanos, downloaded) -> { + lookupStarted.countDown(); + try { + releaseLookup.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + }); + return null; + }); + assertThat(lookupStarted.await(30, TimeUnit.SECONDS)).isTrue(); + TestingLakeTableLookuper oldLookuper = manager.createdLookupers.get(0); + manager.invalidateTableLookuper(PARTITION_TABLE_ID); + lookup(manager, PARTITION_TABLE_INFO); + assertThat(manager.createdCacheNamespaces).doesNotHaveDuplicates(); + assertThat(manager.createdLookupers).hasSize(2); + doAnswer( + invocation -> { + assertThat(manager.createdLookupers) + .allMatch(lookuper -> lookuper.closed); + return null; + }) + .when(manager.sharedManager) + .close(); + + manager.close(); + manager.close(); + assertThat(oldLookuper.closed).isFalse(); + assertThat(manager.createdLookupers.get(1).closed).isTrue(); + verify(manager.sharedManager, never()).close(); + assertThatThrownBy(() -> lookup(manager, PARTITION_TABLE_INFO)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("closed"); + assertThatThrownBy(() -> manager.reconfigure(conf())) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("closed"); + releaseLookup.countDown(); + lookup.get(30, TimeUnit.SECONDS); + verify(manager.sharedManager).close(); + assertThat(manager.createdLookupers).allMatch(lookuper -> lookuper.closed); + assertThat(manager.hasLookuperManager()).isFalse(); + } finally { + releaseLookup.countDown(); + executor.shutdownNow(); + manager.close(); + } } @Test @@ -369,8 +467,9 @@ void testReconfiguresLakePropertiesAndInvalidatesLookuper() throws Exception { initialConf.setString("datalake.paimon.warehouse", "old-warehouse"); TestingHistoricalLakeLookupManager manager = new TestingHistoricalLakeLookupManager(initialConf); - assertThat(manager.hasLookupRuntime()).isTrue(); + assertThat(manager.hasLookuperManager()).isFalse(); manager.startup(NO_OP_SCHEDULER); + assertThat(manager.hasLookuperManager()).isTrue(); lookup(manager, PARTITION_TABLE_INFO); TestingLakeTableLookuper initialLookuper = manager.createdLookupers.get(0); @@ -431,6 +530,15 @@ private static TableInfo tableInfo(long tableId, int schemaId) { private static void lookup( HistoricalLakeLookupManager manager, TableInfo tableInfo, SchemaInfo schemaInfo) throws Exception { + lookup(manager, tableInfo, schemaInfo, NO_OP_LOOKUP_METRIC_RECORDER); + } + + private static void lookup( + HistoricalLakeLookupManager manager, + TableInfo tableInfo, + SchemaInfo schemaInfo, + LakeTableLookuper.LookupMetricRecorder recorder) + throws Exception { TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), 1L, 0); LookupDataForBucket lookupData = lookupData(tableBucket); manager.lookup( @@ -439,7 +547,7 @@ private static void lookup( schemaInfo, ResolvedPartitionSpec.fromPartitionName( tableInfo.getPartitionKeys(), lookupData.originalPartitionName()), - NO_OP_LOOKUP_METRIC_RECORDER); + recorder); } private static final class TestingHistoricalLakeLookupManager @@ -450,6 +558,7 @@ private static final class TestingHistoricalLakeLookupManager private final List createdCacheNamespaces = new ArrayList<>(); private final List createdClusterConfigs = new ArrayList<>(); private final long lookupCacheFileBytes; + private final LakeTableLookuperManager sharedManager = mock(LakeTableLookuperManager.class); private TestingHistoricalLakeLookupManager(Configuration conf) { super( @@ -493,6 +602,16 @@ private TestingHistoricalLakeLookupManager( this.lookupCacheFileBytes = lookupCacheFileBytes; } + @Override + LakeTableLookuperManager createLookuperManager(Configuration configuration) { + File lookupDir = + FlussPaths.historicalLookupRootDir( + new File(configuration.get(ConfigOptions.DATA_DIR))); + assertThat(lookupDir).isDirectory(); + assertThat(new File(lookupDir, "stale-lookup-file")).doesNotExist(); + return sharedManager; + } + @Override LakeTableLookuper createLakeTableLookuper( TablePath tablePath, diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index 3f0dcaacc81..3e7e986d03c 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -105,6 +105,8 @@ The logging-related environment options (`env.log.dir`, `env.log.level`, `env.lo | server.data-disk.write-recover-ratio | Double | 0.80 | Resume writes when the tablet server data disk usage reaches or drops below this ratio. The valid range is `(0.0, server.data-disk.write-limit-ratio)`. This configuration can be updated dynamically without server restart. | | server.data-disk.check-interval | Duration | 30s | The interval at which the tablet server samples the local data disk usage for the write-protection state machine. A shorter interval narrows the time window during which writes can still flow in after the disk crosses the limit ratio, at the cost of slightly more frequent `statvfs` calls (which are in-memory and cheap). The default 30s is suitable for typical write workloads. | | server.historical-partition.thread-pool.max-size | Integer | 10 | The maximum number of threads used for historical partition operations, such as lake lookups and writes. This configuration can be updated dynamically without server restart. | +| server.historical-partition.lookup-cache.max-disk-ratio | Double | 0.10 | The maximum fraction of the volume containing the first available data directory used by historical lookup files. All tables share one disk budget, with eviction at file granularity. The valid range is (0.0, 1.0]. This configuration can be updated dynamically. | +| server.historical-partition.lookuper-cache.expire-after-access | Duration | 3 h | Idle expiration for historical table lookupers and their shared file cache. Each cache tracks access independently. Replaces Paimon's table-level `lookup.cache-file-retention` for historical lookups. Dynamic changes apply to both caches without replacing active lookupers. | ## Zookeeper diff --git a/website/docs/maintenance/observability/monitor-metrics.md b/website/docs/maintenance/observability/monitor-metrics.md index ee57382f523..fb88ab0a0a7 100644 --- a/website/docs/maintenance/observability/monitor-metrics.md +++ b/website/docs/maintenance/observability/monitor-metrics.md @@ -616,9 +616,9 @@ Some metrics might not be exposed when using other JVM implementations (e.g. IBM Gauge - lookupCacheCapacityEvictions - The cumulative number of cached table lookupers evicted because the cache retains at most ten tables. - Counter + lookupCacheFileCapacityEvictions + The cumulative number of lookup files evicted to enforce the shared TabletServer disk budget. Expiration, replacement, and explicit invalidation are excluded. + Gauge logicalStorage From e49832952f37a34e468f005ac3ac87c4174f20e3 Mon Sep 17 00:00:00 2001 From: Junfan Zhang Date: Mon, 28 Sep 2026 16:16:19 +0800 Subject: [PATCH 4/8] LookupRuntimeOptions to LookupCacheOptions --- .../fluss/lake/lakestorage/LakeStorage.java | 4 ++-- .../lakestorage/LakeTableLookuperManager.java | 6 ++--- .../lakestorage/PluginLakeStorageWrapper.java | 6 ++--- .../lake/lakestorage/LakeStorageTest.java | 23 +++++++++---------- .../fluss/lake/paimon/PaimonLakeStorage.java | 8 +++---- .../lookup/PaimonLakeTableLookuperTest.java | 6 ++--- .../HistoricalLakeLookupManager.java | 7 +++--- .../lakehouse/TestingPaimonStoragePlugin.java | 6 ++--- .../HistoricalLakeLookupManagerTest.java | 10 ++++---- 9 files changed, 37 insertions(+), 39 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java index 7f2329bdd56..8b45d3a999b 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeStorage.java @@ -18,7 +18,7 @@ package org.apache.fluss.lake.lakestorage; import org.apache.fluss.annotation.PublicEvolving; -import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupRuntimeOptions; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupCacheOptions; import org.apache.fluss.lake.source.LakeSource; import org.apache.fluss.lake.writer.LakeTieringFactory; import org.apache.fluss.metadata.TablePath; @@ -60,7 +60,7 @@ public interface LakeStorage { * @return the lookuper manager */ default LakeTableLookuperManager createLakeTableLookuperManager( - String ioTmpDir, LookupRuntimeOptions options) { + String ioTmpDir, LookupCacheOptions options) { throw new UnsupportedOperationException( "Point lookup is not supported for this lake storage."); } diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuperManager.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuperManager.java index aa0f213cbb5..3e298b77617 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuperManager.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/LakeTableLookuperManager.java @@ -57,7 +57,7 @@ public interface LakeTableLookuperManager extends AutoCloseable { * * @param options the new runtime resource settings */ - void reconfigure(LookupRuntimeOptions options); + void reconfigure(LookupCacheOptions options); /** Returns the cumulative number of lookup files evicted by the shared disk-space budget. */ default long fileCacheCapacityEvictions() { @@ -70,7 +70,7 @@ default long fileCacheCapacityEvictions() { *

These settings apply to resources shared by all table lookupers in one runtime. * Lake-format and table-specific configuration is supplied separately when creating a lookuper. */ - final class LookupRuntimeOptions { + final class LookupCacheOptions { private final long localCacheMaxBytes; private final Duration expireAfterAccess; @@ -83,7 +83,7 @@ final class LookupRuntimeOptions { * this budget * @param expireAfterAccess positive idle expiration for individual cached lookup files */ - public LookupRuntimeOptions(long localCacheMaxBytes, Duration expireAfterAccess) { + public LookupCacheOptions(long localCacheMaxBytes, Duration expireAfterAccess) { checkArgument(localCacheMaxBytes > 0, "localCacheMaxBytes must be greater than 0."); this.localCacheMaxBytes = localCacheMaxBytes; this.expireAfterAccess = diff --git a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java index 5cbfa194bea..c9885707e55 100644 --- a/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java +++ b/fluss-common/src/main/java/org/apache/fluss/lake/lakestorage/PluginLakeStorageWrapper.java @@ -20,7 +20,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.TableAlreadyExistException; import org.apache.fluss.exception.TableNotExistException; -import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupRuntimeOptions; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupCacheOptions; import org.apache.fluss.lake.source.LakeSource; import org.apache.fluss.lake.writer.LakeTieringFactory; import org.apache.fluss.metadata.TableChange; @@ -141,7 +141,7 @@ public LakeSource createLakeSource(TablePath tablePath) { @Override public LakeTableLookuperManager createLakeTableLookuperManager( - String ioTmpDir, LookupRuntimeOptions options) { + String ioTmpDir, LookupCacheOptions options) { try (TemporaryClassLoaderContext ignored = TemporaryClassLoaderContext.of(loader)) { return new ClassLoaderFixingLakeTableLookuperManager( inner.createLakeTableLookuperManager(ioTmpDir, options), loader); @@ -170,7 +170,7 @@ public LakeTableLookuper createLakeTableLookuper(TablePath tablePath, Context co } @Override - public void reconfigure(LookupRuntimeOptions options) { + public void reconfigure(LookupCacheOptions options) { try (TemporaryClassLoaderContext ignored = TemporaryClassLoaderContext.of(loader)) { inner.reconfigure(options); } diff --git a/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java b/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java index c8e14218a20..e5d6a2f462b 100644 --- a/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeStorageTest.java @@ -20,7 +20,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.TableAlreadyExistException; import org.apache.fluss.exception.TableNotExistException; -import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupRuntimeOptions; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupCacheOptions; import org.apache.fluss.lake.source.LakeSource; import org.apache.fluss.lake.writer.LakeTieringFactory; import org.apache.fluss.metadata.TableChange; @@ -45,14 +45,14 @@ class LakeStorageTest { private static final String TEST_LAKE_PLUGIN_FORMAT = "test-plugin"; @Test - void testLookupRuntimeOptionsRejectInvalidResourceLimits() { - assertThatThrownBy(() -> new LookupRuntimeOptions(0L, Duration.ofHours(1))) + void testLookupCacheOptionsRejectInvalidResourceLimits() { + assertThatThrownBy(() -> new LookupCacheOptions(0L, Duration.ofHours(1))) .isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> new LookupRuntimeOptions(1024L, Duration.ZERO)) + assertThatThrownBy(() -> new LookupCacheOptions(1024L, Duration.ZERO)) .isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> new LookupRuntimeOptions(1024L, Duration.ofSeconds(-1))) + assertThatThrownBy(() -> new LookupCacheOptions(1024L, Duration.ofSeconds(-1))) .isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> new LookupRuntimeOptions(1024L, null)) + assertThatThrownBy(() -> new LookupCacheOptions(1024L, null)) .isInstanceOf(NullPointerException.class); } @@ -113,7 +113,7 @@ void testWithPluginManager() throws Exception { LakeTableLookuperManager lookuperManager = lakeStorage.createLakeTableLookuperManager( - "lookup-dir", new LookupRuntimeOptions(1024L, Duration.ofHours(3))); + "lookup-dir", new LookupCacheOptions(1024L, Duration.ofHours(3))); assertThat(lookuperManager) .isInstanceOf( PluginLakeStorageWrapper.ClassLoaderFixingLakeTableLookuperManager.class); @@ -122,8 +122,7 @@ void testWithPluginManager() throws Exception { ((PluginLakeStorageWrapper.ClassLoaderFixingLakeTableLookuperManager) lookuperManager) .getWrappedDelegate(); - LookupRuntimeOptions updatedOptions = - new LookupRuntimeOptions(2048L, Duration.ofMinutes(30)); + LookupCacheOptions updatedOptions = new LookupCacheOptions(2048L, Duration.ofMinutes(30)); lookuperManager.reconfigure(updatedOptions); assertThat(innerLookuperManager.options).isSameAs(updatedOptions); assertThat(lookuperManager.fileCacheCapacityEvictions()).isEqualTo(3L); @@ -181,7 +180,7 @@ public LakeSource createLakeSource(TablePath tablePath) { @Override public LakeTableLookuperManager createLakeTableLookuperManager( - String ioTmpDir, LookupRuntimeOptions options) { + String ioTmpDir, LookupCacheOptions options) { return new TestLakeTableLookuperManager(); } } @@ -189,7 +188,7 @@ public LakeTableLookuperManager createLakeTableLookuperManager( private static class TestLakeTableLookuperManager implements LakeTableLookuperManager { private boolean closed; - private LookupRuntimeOptions options; + private LookupCacheOptions options; @Override public LakeTableLookuper createLakeTableLookuper(TablePath tablePath, Context context) { @@ -197,7 +196,7 @@ public LakeTableLookuper createLakeTableLookuper(TablePath tablePath, Context co } @Override - public void reconfigure(LookupRuntimeOptions options) { + public void reconfigure(LookupCacheOptions options) { this.options = options; } diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java index 349996ebf74..e121878d3f2 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/PaimonLakeStorage.java @@ -21,7 +21,7 @@ import org.apache.fluss.lake.lakestorage.LakeStorage; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager; -import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupRuntimeOptions; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupCacheOptions; import org.apache.fluss.lake.paimon.lookup.PaimonLakeTableLookuper; import org.apache.fluss.lake.paimon.lookup.PaimonScanBasedTableLookuper; import org.apache.fluss.lake.paimon.lookup.SharedLookupFileCache; @@ -67,7 +67,7 @@ public LakeSource createLakeSource(TablePath tablePath) { @Override public LakeTableLookuperManager createLakeTableLookuperManager( - String ioTmpDir, LookupRuntimeOptions options) { + String ioTmpDir, LookupCacheOptions options) { return new PaimonLakeTableLookuperManager(ioTmpDir, options); } @@ -76,7 +76,7 @@ private static final class PaimonLakeTableLookuperManager implements LakeTableLo private final IOManager ioManager; private final SharedLookupFileCache lookupFileCache; - private PaimonLakeTableLookuperManager(String ioTmpDir, LookupRuntimeOptions options) { + private PaimonLakeTableLookuperManager(String ioTmpDir, LookupCacheOptions options) { checkNotNull(options, "options must not be null."); this.ioManager = IOManager.create(checkNotNull(ioTmpDir, "ioTmpDir must not be null.")); this.lookupFileCache = @@ -104,7 +104,7 @@ public LakeTableLookuper createLakeTableLookuper(TablePath tablePath, Context co } @Override - public void reconfigure(LookupRuntimeOptions options) { + public void reconfigure(LookupCacheOptions options) { checkNotNull(options, "options must not be null."); lookupFileCache.updateMaxDiskSize(new MemorySize(options.localCacheMaxBytes())); lookupFileCache.updateExpireAfterAccess(options.expireAfterAccess()); diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java index 80af27310ac..073520efa83 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java @@ -27,7 +27,7 @@ import org.apache.fluss.exception.RetriableException; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager; -import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupRuntimeOptions; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupCacheOptions; import org.apache.fluss.lake.lakestorage.TestingLakeCatalogContext; import org.apache.fluss.lake.paimon.PaimonLakeCatalog; import org.apache.fluss.lake.paimon.PaimonLakeStorage; @@ -134,7 +134,7 @@ void setUp() { new PaimonLakeStorage(paimonConfig) .createLakeTableLookuperManager( tempWarehouseDir.getAbsolutePath(), - new LookupRuntimeOptions( + new LookupCacheOptions( LOOKUP_CACHE_MAX_DISK_BYTES, Duration.ofHours(3))); } @@ -504,7 +504,7 @@ void testSharesIOManagerAndDeletesOnlyClosedLookuperFiles() throws Exception { new PaimonLakeStorage(paimonConfig) .createLakeTableLookuperManager( lookupDir.getAbsolutePath(), - new LookupRuntimeOptions( + new LookupCacheOptions( LOOKUP_CACHE_MAX_DISK_BYTES, Duration.ofHours(3))); try { try (LakeTableLookuper firstLookuper = diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java index 13a0bacf2c0..1575a02a4a5 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java @@ -28,7 +28,7 @@ import org.apache.fluss.lake.lakestorage.LakeStoragePluginSetUp; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager; -import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupRuntimeOptions; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupCacheOptions; import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.LakeLookupMode; import org.apache.fluss.metadata.ResolvedPartitionSpec; @@ -365,8 +365,7 @@ void reconfigure(Configuration newConf) { lookuperManager = createLookuperManager(newConf); } if ((cacheLimitChanged || expirationChanged) && lookuperManager != null) { - lookuperManager.reconfigure( - new LookupRuntimeOptions(newMaxDiskBytes, newExpiration)); + lookuperManager.reconfigure(new LookupCacheOptions(newMaxDiskBytes, newExpiration)); } // Publish the configuration before its version. A lookup that observes the new version // must also observe the matching configuration snapshot. @@ -475,7 +474,7 @@ LakeTableLookuperManager createLookuperManager(Configuration configuration) { lakeStoragePlugin.createLakeStorage(Configuration.fromMap(lakeProperties)); return lakeStorage.createLakeTableLookuperManager( historicalLookupCacheRootDir.getAbsolutePath(), - new LookupRuntimeOptions( + new LookupCacheOptions( lookupCacheMaxDiskBytes, configuration.get( ConfigOptions diff --git a/fluss-server/src/test/java/org/apache/fluss/server/lakehouse/TestingPaimonStoragePlugin.java b/fluss-server/src/test/java/org/apache/fluss/server/lakehouse/TestingPaimonStoragePlugin.java index 5eb3d982864..84c12868f0e 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/lakehouse/TestingPaimonStoragePlugin.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/lakehouse/TestingPaimonStoragePlugin.java @@ -30,7 +30,7 @@ import org.apache.fluss.lake.lakestorage.LakeStoragePlugin; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager; -import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupRuntimeOptions; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupCacheOptions; import org.apache.fluss.lake.serializer.SimpleVersionedSerializer; import org.apache.fluss.lake.source.LakeSource; import org.apache.fluss.lake.writer.LakeTieringFactory; @@ -92,7 +92,7 @@ public LakeSource createLakeSource(TablePath tablePath) { @Override public LakeTableLookuperManager createLakeTableLookuperManager( - String ioTmpDir, LookupRuntimeOptions options) { + String ioTmpDir, LookupCacheOptions options) { return new LakeTableLookuperManager() { @Override public LakeTableLookuper createLakeTableLookuper( @@ -101,7 +101,7 @@ public LakeTableLookuper createLakeTableLookuper( } @Override - public void reconfigure(LookupRuntimeOptions options) {} + public void reconfigure(LookupCacheOptions options) {} @Override public void close() {} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java index 507786f2f4e..f2b5af6dc59 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java @@ -23,7 +23,7 @@ import org.apache.fluss.config.TableConfig; import org.apache.fluss.lake.lakestorage.LakeTableLookuper; import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager; -import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupRuntimeOptions; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupCacheOptions; import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.KvFormat; import org.apache.fluss.metadata.LakeLookupMode; @@ -375,8 +375,8 @@ void testUpdatesSharedCacheLimitWithoutReplacingLookuper() throws Exception { assertThat(manager.lookupCacheMaxDiskBytes()).isEqualTo(20L); assertThat(lookuper.closed).isFalse(); assertThat(manager.cachedTableCount()).isOne(); - ArgumentCaptor options = - ArgumentCaptor.forClass(LookupRuntimeOptions.class); + ArgumentCaptor options = + ArgumentCaptor.forClass(LookupCacheOptions.class); verify(manager.sharedManager).reconfigure(options.capture()); assertThat(options.getValue().localCacheMaxBytes()).isEqualTo(20L); assertThat(options.getValue().expireAfterAccess()).isEqualTo(Duration.ofHours(3)); @@ -388,8 +388,8 @@ void testUpdatesFileCacheExpirationWithoutReplacingLookuper() throws Exception { lookup(manager, PARTITION_TABLE_INFO); manager.reconfigure(confWithExpiration(Duration.ofMinutes(30))); - ArgumentCaptor options = - ArgumentCaptor.forClass(LookupRuntimeOptions.class); + ArgumentCaptor options = + ArgumentCaptor.forClass(LookupCacheOptions.class); verify(manager.sharedManager).reconfigure(options.capture()); assertThat(options.getValue().expireAfterAccess()).isEqualTo(Duration.ofMinutes(30)); assertThat(manager.createdLookupers).hasSize(1).noneMatch(lookuper -> lookuper.closed); From 996f63c887243da9b01a2e40a7fbdf574ea87434 Mon Sep 17 00:00:00 2001 From: Junfan Zhang Date: Mon, 28 Sep 2026 16:25:00 +0800 Subject: [PATCH 5/8] better lock --- .../HistoricalLakeLookupManager.java | 128 ++++++++---------- .../HistoricalLakeLookupManagerTest.java | 18 +-- 2 files changed, 62 insertions(+), 84 deletions(-) diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java index 1575a02a4a5..d5b85af442f 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManager.java @@ -66,7 +66,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Stream; @@ -117,11 +116,9 @@ class HistoricalLakeLookupManager implements AutoCloseable { private volatile long lakeConfigVersion; private final @Nullable PluginManager pluginManager; private volatile @Nullable LakeTableLookuperManager lookuperManager; - // Acquisitions may run concurrently. Closing first prevents any new cache entries, then - // invalidates existing ones without holding this lock. In-flight lookups drain independently. + // Acquisitions may run concurrently. Lifecycle changes exclude new acquisitions, while + // in-flight lookups release their table lookupers independently. private final ReentrantReadWriteLock lifecycleLock = new ReentrantReadWriteLock(); - // Includes invalidated lookupers until their close operation has actually finished. - private final AtomicInteger openLookupers = new AtomicInteger(); private final AtomicLong lookuperIdSequence; private final Cache lakeTableLookupers; private final ConcurrentMap requiredLakeSnapshotIds = new ConcurrentHashMap<>(); @@ -211,35 +208,40 @@ private static com.github.benmanes.caffeine.cache.Scheduler createCacheScheduler *

The cache root under this server's first data directory is removed and recreated before * lookups are accepted. */ - synchronized void startup(Scheduler scheduler) { + void startup(Scheduler scheduler) { checkNotNull(scheduler, "scheduler must not be null."); - checkState(!closed, "Historical lake lookup manager is closed."); - if (started) { - return; - } - try { - FileUtils.deleteDirectory(historicalLookupCacheRootDir); - } catch (IOException e) { - LOG.warn( - "Failed to clean historical lookup cache directory {}.", - historicalLookupCacheRootDir, - e); - } + lifecycleLock.writeLock().lock(); try { - Files.createDirectories(historicalLookupCacheRootDir.toPath()); - } catch (IOException e) { - throw new FlussRuntimeException( - "Failed to create historical lookup cache directory: " - + historicalLookupCacheRootDir, - e); + checkState(!closed, "Historical lake lookup manager is closed."); + if (started) { + return; + } + try { + FileUtils.deleteDirectory(historicalLookupCacheRootDir); + } catch (IOException e) { + LOG.warn( + "Failed to clean historical lookup cache directory {}.", + historicalLookupCacheRootDir, + e); + } + try { + Files.createDirectories(historicalLookupCacheRootDir.toPath()); + } catch (IOException e) { + throw new FlussRuntimeException( + "Failed to create historical lookup cache directory: " + + historicalLookupCacheRootDir, + e); + } + lookuperManager = createLookuperManager(conf); + scheduler.schedule( + LOOKUP_CACHE_DISK_SIZE_TASK_NAME, + this::updateLookupCacheDiskSize, + 0L, + LOOKUP_CACHE_DISK_SIZE_CHECK_INTERVAL.toMillis()); + started = true; + } finally { + lifecycleLock.writeLock().unlock(); } - lookuperManager = createLookuperManager(conf); - scheduler.schedule( - LOOKUP_CACHE_DISK_SIZE_TASK_NAME, - this::updateLookupCacheDiskSize, - 0L, - LOOKUP_CACHE_DISK_SIZE_CHECK_INTERVAL.toMillis()); - started = true; } /** Looks up a batch of keys from one historical lake partition. */ @@ -274,33 +276,25 @@ List lookup( @Override public void close() { + LakeTableLookuperManager manager; lifecycleLock.writeLock().lock(); try { - synchronized (this) { - if (closed) { - return; - } - closed = true; - } - } finally { - lifecycleLock.writeLock().unlock(); - } - lakeTableLookupers.invalidateAll(); - lakeTableLookupers.cleanUp(); - requiredLakeSnapshotIds.clear(); - closeSharedResourcesIfUnused(); - } - - private void closeSharedResourcesIfUnused() { - LakeTableLookuperManager manager; - synchronized (this) { - if (!closed || openLookupers.get() != 0) { + if (closed) { return; } + closed = true; manager = lookuperManager; lookuperManager = null; + } finally { + lifecycleLock.writeLock().unlock(); + } + try { + lakeTableLookupers.invalidateAll(); + lakeTableLookupers.cleanUp(); + requiredLakeSnapshotIds.clear(); + } finally { + IOUtils.closeQuietly(manager, "historical lake lookuper manager"); } - IOUtils.closeQuietly(manager, "historical lake lookuper manager"); } /** Invalidates the cached lake lookuper for the given table. */ @@ -345,7 +339,8 @@ void reconfigure(Configuration newConf) { newConf.get( ConfigOptions .SERVER_HISTORICAL_PARTITION_LOOKUPER_CACHE_EXPIRE_AFTER_ACCESS); - synchronized (this) { + lifecycleLock.writeLock().lock(); + try { checkState(!closed, "Historical lake lookup manager is closed."); long newMaxDiskBytes = cacheBytes( @@ -373,18 +368,19 @@ void reconfigure(Configuration newConf) { if (lakeConfigChanged) { lakeConfigVersion++; } - } - if (expirationChanged) { - lakeTableLookupers - .policy() - .expireAfterAccess() - .get() - .setExpiresAfter(newExpiration.toMillis(), TimeUnit.MILLISECONDS); + if (expirationChanged) { + lakeTableLookupers + .policy() + .expireAfterAccess() + .get() + .setExpiresAfter(newExpiration.toMillis(), TimeUnit.MILLISECONDS); + } + } finally { + lifecycleLock.writeLock().unlock(); } if (lakeConfigChanged) { - // Do not invalidate while holding this monitor: closing a lookuper may need it to - // release shared resources. Inactive lookupers close immediately; active lookupers - // close after their last lookup releases them. + // Inactive lookupers close immediately; active lookupers close after their last lookup + // releases them. lakeTableLookupers.invalidateAll(); lakeTableLookupers.cleanUp(); } @@ -536,12 +532,7 @@ private static long fileSize(Path path) { } private void closeLookuper(CachedLakeTableLookuper cachedLookuper) { - try { - IOUtils.closeQuietly(cachedLookuper.lookuper, "historical lake table lookuper"); - } finally { - openLookupers.decrementAndGet(); - closeSharedResourcesIfUnused(); - } + IOUtils.closeQuietly(cachedLookuper.lookuper, "historical lake table lookuper"); } private CachedLakeTableLookuper acquireLookuper(LookupContext context, TableInfo tableInfo) { @@ -639,7 +630,6 @@ private CachedLakeTableLookuper( this.lakeConfigVersion = lakeConfigVersion; this.lakeSnapshotId = lakeSnapshotId; this.lookuper = lookuper; - openLookupers.incrementAndGet(); } private synchronized void acquire(@Nullable Long requiredLakeSnapshotId) { diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java index f2b5af6dc59..480bcbe0042 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/historical/HistoricalLakeLookupManagerTest.java @@ -64,9 +64,7 @@ import static org.apache.fluss.record.TestData.PARTITION_TABLE_INFO; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -398,7 +396,7 @@ void testUpdatesFileCacheExpirationWithoutReplacingLookuper() throws Exception { } @Test - void testClosesSharedResourcesAfterInvalidatedLookuperFinishes() throws Exception { + void testClosesSharedResourcesWhileInvalidatedLookuperIsActive() throws Exception { TestingHistoricalLakeLookupManager manager = createTestingManager(); CountDownLatch lookupStarted = new CountDownLatch(1); CountDownLatch releaseLookup = new CountDownLatch(1); @@ -428,20 +426,12 @@ void testClosesSharedResourcesAfterInvalidatedLookuperFinishes() throws Exceptio lookup(manager, PARTITION_TABLE_INFO); assertThat(manager.createdCacheNamespaces).doesNotHaveDuplicates(); assertThat(manager.createdLookupers).hasSize(2); - doAnswer( - invocation -> { - assertThat(manager.createdLookupers) - .allMatch(lookuper -> lookuper.closed); - return null; - }) - .when(manager.sharedManager) - .close(); - manager.close(); manager.close(); assertThat(oldLookuper.closed).isFalse(); assertThat(manager.createdLookupers.get(1).closed).isTrue(); - verify(manager.sharedManager, never()).close(); + verify(manager.sharedManager).close(); + assertThat(manager.hasLookuperManager()).isFalse(); assertThatThrownBy(() -> lookup(manager, PARTITION_TABLE_INFO)) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("closed"); @@ -450,9 +440,7 @@ void testClosesSharedResourcesAfterInvalidatedLookuperFinishes() throws Exceptio .hasMessageContaining("closed"); releaseLookup.countDown(); lookup.get(30, TimeUnit.SECONDS); - verify(manager.sharedManager).close(); assertThat(manager.createdLookupers).allMatch(lookuper -> lookuper.closed); - assertThat(manager.hasLookuperManager()).isFalse(); } finally { releaseLookup.countDown(); executor.shutdownNow(); From 656508082b8abc4ca7a454c2071cb7065ed9167a Mon Sep 17 00:00:00 2001 From: Junfan Zhang Date: Mon, 28 Sep 2026 16:27:49 +0800 Subject: [PATCH 6/8] add todo for upstream paimon support this feature --- .../apache/fluss/lake/paimon/lookup/PaimonLocalTableQuery.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLocalTableQuery.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLocalTableQuery.java index 1dd75fc1290..31122772e25 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLocalTableQuery.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLocalTableQuery.java @@ -73,6 +73,9 @@ * {@link LookupLevels}. This replaces the lazy, query-owned cache built from table retention and * disk-size options. Closing a query only invalidates its namespace; the shared cache and I/O * manager belong to the lake table lookuper manager. + * + *

TODO: Once Paimon's {@code LocalTableQuery} supports an externally managed shared lookup-file + * cache, use Paimon's implementation directly and remove this copy. */ final class PaimonLocalTableQuery implements TableQuery { From 5336fdc1be5147f193abeb2eacae52f997556511 Mon Sep 17 00:00:00 2001 From: Junfan Zhang Date: Mon, 28 Sep 2026 16:38:01 +0800 Subject: [PATCH 7/8] add missing tests --- .../LakeTableLookuperManagerTest.java | 87 ++++++++++++++++++ .../lookup/SharedLookupFileCacheTest.java | 89 +++++++++++++++++++ 2 files changed, 176 insertions(+) create mode 100644 fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeTableLookuperManagerTest.java diff --git a/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeTableLookuperManagerTest.java b/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeTableLookuperManagerTest.java new file mode 100644 index 00000000000..1c70bdfdad4 --- /dev/null +++ b/fluss-common/src/test/java/org/apache/fluss/lake/lakestorage/LakeTableLookuperManagerTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.lake.lakestorage; + +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.TableConfig; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.Context; +import org.apache.fluss.lake.lakestorage.LakeTableLookuperManager.LookupCacheOptions; +import org.apache.fluss.metadata.TablePath; + +import org.junit.jupiter.api.Test; + +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests the shared lake table lookuper manager contract. */ +class LakeTableLookuperManagerTest { + + @Test + void testContextCarriesLookuperSettings() { + Configuration lakeConfiguration = new Configuration(); + TableConfig tableConfig = new TableConfig(new Configuration()); + AtomicBoolean diskWriteChecked = new AtomicBoolean(); + Runnable diskWriteGuard = () -> diskWriteChecked.set(true); + + Context context = new Context(lakeConfiguration, "table-1", tableConfig, diskWriteGuard); + + assertThat(context.lakeConfiguration()).isSameAs(lakeConfiguration); + assertThat(context.cacheNamespace()).isEqualTo("table-1"); + assertThat(context.tableConfig()).isSameAs(tableConfig); + context.diskWriteGuard().run(); + assertThat(diskWriteChecked.get()).isTrue(); + } + + @Test + void testContextRejectsMissingSettings() { + Configuration lakeConfiguration = new Configuration(); + TableConfig tableConfig = new TableConfig(new Configuration()); + Runnable diskWriteGuard = () -> {}; + + assertThatThrownBy(() -> new Context(null, "table-1", tableConfig, diskWriteGuard)) + .isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> new Context(lakeConfiguration, null, tableConfig, diskWriteGuard)) + .isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> new Context(lakeConfiguration, "table-1", null, diskWriteGuard)) + .isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> new Context(lakeConfiguration, "table-1", tableConfig, null)) + .isInstanceOf(NullPointerException.class); + } + + @Test + void testDefaultCapacityEvictionsMetricIsZero() { + LakeTableLookuperManager manager = + new LakeTableLookuperManager() { + @Override + public LakeTableLookuper createLakeTableLookuper( + TablePath tablePath, Context context) { + return null; + } + + @Override + public void reconfigure(LookupCacheOptions options) {} + + @Override + public void close() {} + }; + + assertThat(manager.fileCacheCapacityEvictions()).isZero(); + } +} diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/SharedLookupFileCacheTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/SharedLookupFileCacheTest.java index f38d2b4d7b4..827e38b3ad5 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/SharedLookupFileCacheTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/SharedLookupFileCacheTest.java @@ -29,7 +29,11 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.time.Duration; +import java.util.AbstractMap; import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -39,6 +43,7 @@ import java.util.concurrent.atomic.AtomicLong; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Tests for {@link SharedLookupFileCache}. */ class SharedLookupFileCacheTest { @@ -103,6 +108,90 @@ void testGlobalBudgetAcrossNamespaces() throws Exception { } } + @Test + void testNamespacedMapKeepsMutationsWithinItsNamespace() throws Exception { + File firstFile = lookupFile("map-first.lookup"); + File secondFile = lookupFile("map-second.lookup"); + File replacementFile = lookupFile("map-replacement.lookup"); + File finalFile = lookupFile("map-final.lookup"); + File additionalFile = lookupFile("map-additional.lookup"); + + try (SharedLookupFileCache sharedCache = + new SharedLookupFileCache(Duration.ofHours(1), MemorySize.ofKibiBytes(8))) { + ConcurrentMap firstMap = sharedCache.namespaced("first").asMap(); + ConcurrentMap secondMap = sharedCache.namespaced("second").asMap(); + LookupFile firstLookupFile = lookupFile(firstFile); + LookupFile secondLookupFile = lookupFile(secondFile); + LookupFile replacementLookupFile = lookupFile(replacementFile); + LookupFile finalLookupFile = lookupFile(finalFile); + + assertThat(firstMap.putIfAbsent("same", firstLookupFile)).isNull(); + assertThat(secondMap.put("same", secondLookupFile)).isNull(); + assertThat(firstMap.get("same")).isSameAs(firstLookupFile); + assertThat(firstMap.get(1)).isNull(); + assertThat(firstMap.entrySet()) + .containsOnly(new AbstractMap.SimpleImmutableEntry<>("same", firstLookupFile)); + + assertThat(firstMap.replace("same", secondLookupFile, replacementLookupFile)).isFalse(); + assertThat(firstMap.replace("same", firstLookupFile, replacementLookupFile)).isTrue(); + assertThat(firstFile).doesNotExist(); + assertThat(firstMap.replace("same", finalLookupFile)).isSameAs(replacementLookupFile); + assertThat(replacementFile).doesNotExist(); + assertThat(firstMap.putIfAbsent("same", firstLookupFile)).isSameAs(finalLookupFile); + assertThat(firstMap.remove("same", secondLookupFile)).isFalse(); + assertThat(firstMap.remove("same", finalLookupFile)).isTrue(); + assertThat(finalFile).doesNotExist(); + assertThat(firstMap.remove("same")).isNull(); + + assertThat(firstMap.put("additional", lookupFile(additionalFile))).isNull(); + firstMap.clear(); + assertThat(additionalFile).doesNotExist(); + assertThat(secondMap.get("same")).isSameAs(secondLookupFile); + assertThat(secondFile).exists(); + } + + assertThat(secondFile).doesNotExist(); + } + + @Test + void testNamespacedCacheBulkOperationsAndPolicy() throws Exception { + File bulkFile = lookupFile("bulk.lookup"); + File createdFile = lookupFile("created.lookup"); + File otherFile = lookupFile("other.lookup"); + + try (SharedLookupFileCache sharedCache = + new SharedLookupFileCache(Duration.ofHours(1), MemorySize.ofKibiBytes(8))) { + Cache first = sharedCache.namespaced("first"); + Cache second = sharedCache.namespaced("second"); + LookupFile bulkLookupFile = lookupFile(bulkFile); + LookupFile createdLookupFile = lookupFile(createdFile); + LookupFile otherLookupFile = lookupFile(otherFile); + first.putAll(Collections.singletonMap("bulk", bulkLookupFile)); + second.put("bulk", otherLookupFile); + + assertThat(first.get("created", ignored -> createdLookupFile)) + .isSameAs(createdLookupFile); + Map present = + first.getAllPresent(Arrays.asList("bulk", "created", "missing", 1)); + assertThat(present) + .containsOnlyKeys("bulk", "created") + .containsEntry("bulk", bulkLookupFile) + .containsEntry("created", createdLookupFile); + assertThat(first.getIfPresent(1)).isNull(); + assertThat(first.stats()).isNotNull(); + assertThatThrownBy(first::policy).isInstanceOf(UnsupportedOperationException.class); + + first.invalidate("bulk"); + assertThat(bulkFile).doesNotExist(); + first.invalidateAll(Collections.singletonList("created")); + assertThat(createdFile).doesNotExist(); + assertThat(second.getIfPresent("bulk")).isSameAs(otherLookupFile); + assertThat(otherFile).exists(); + } + + assertThat(otherFile).doesNotExist(); + } + @Test void testDynamicExpirationAndIndependentFileAccess() throws Exception { AtomicLong time = new AtomicLong(); From 027ba5e7db647d723f7c173dbd8769fa994c7d37 Mon Sep 17 00:00:00 2001 From: Junfan Zhang Date: Mon, 28 Sep 2026 16:50:02 +0800 Subject: [PATCH 8/8] [server] Add support of concurrent SST lookup for paimon historical partition --- .../paimon/lookup/PaimonLakeTableLookuper.java | 14 +++----------- .../lake/paimon/lookup/PaimonLocalTableQuery.java | 10 +++++++++- .../paimon/lookup/PaimonLakeTableLookuperTest.java | 8 +++----- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java index 34e0103d159..5d86050855f 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuper.java @@ -92,8 +92,8 @@ * registered partition-bucket and updates its file set in place. Paimon keeps lookup files for data * files that remain active and lazily downloads lookup files only for newly added data files. * - *

Calls to {@link PaimonLocalTableQuery#lookup} are serialized because Paimon 2.0 shares mutable - * lookup-store comparator state across local lookup files. + *

Calls to {@link PaimonLocalTableQuery#lookup} can run concurrently. The local query isolates + * Paimon 2.0's mutable lookup-store comparator by thread. * *

Close is expected only after the owner has drained active lookups. It is synchronized with * lazy initialization and file-set updates, but deliberately does not add a lifecycle lock to every @@ -114,7 +114,6 @@ public class PaimonLakeTableLookuper implements LakeTableLookuper { private final Runnable diskWriteGuard; private final ThreadLocal lookupFileDownloaded; - private final Object paimonLookupLock; // Guards lazy initialization, close, and registered file-set updates. private final Object lookupStateLock; private final Map> registeredFiles; @@ -152,7 +151,6 @@ public PaimonLakeTableLookuper( this.tableConfig = checkNotNull(tableConfig, "tableConfig must not be null."); this.diskWriteGuard = checkNotNull(diskWriteGuard, "diskWriteGuard must not be null."); this.lookupFileDownloaded = new ThreadLocal<>(); - this.paimonLookupLock = new Object(); this.lookupStateLock = new Object(); this.ioManager = new TrackingIOManager(checkNotNull(ioManager, "ioManager must not be null.")); @@ -474,13 +472,7 @@ private byte[] deriveBucketKey(byte[] key, LookupContext context) { int bucket, org.apache.paimon.data.InternalRow key) throws IOException { - // TODO: Remove this lock once https://github.com/apache/paimon/issues/9483 is fixed in the - // Paimon version used by Fluss. If concurrent lookup is needed sooner, bring Paimon's - // LocalTableQuery implementation into Fluss and make it thread-safe, following the approach - // in https://github.com/apache/fluss/pull/4113. - synchronized (paimonLookupLock) { - return localTableQuery.lookup(partition, bucket, key); - } + return localTableQuery.lookup(partition, bucket, key); } private List getOrInitializeFiles(PaimonPartitionBucket partitionBucket) { diff --git a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLocalTableQuery.java b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLocalTableQuery.java index 31122772e25..9dbfe48142a 100644 --- a/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLocalTableQuery.java +++ b/fluss-lake/fluss-lake-paimon/src/main/java/org/apache/fluss/lake/paimon/lookup/PaimonLocalTableQuery.java @@ -34,6 +34,7 @@ import org.apache.paimon.io.KeyValueFileReaderFactory; import org.apache.paimon.io.cache.CacheManager; import org.apache.paimon.lookup.LookupStoreFactory; +import org.apache.paimon.memory.MemorySlice; import org.apache.paimon.mergetree.Levels; import org.apache.paimon.mergetree.LookupFile; import org.apache.paimon.mergetree.LookupLevels; @@ -123,13 +124,20 @@ final class PaimonLocalTableQuery implements TableQuery { this.fileIO = table.fileIO(); RowType keyType = readerFactoryBuilder.keyType(); this.keyComparatorSupplier = new KeyComparatorSupplier(readerFactoryBuilder.keyType()); + // Paimon 2.0.0's slice comparator reuses mutable RowReaders. The lookup store factory + // shares its comparator across all lookup files, so each lookup thread needs its own. + ThreadLocal> threadLocalComparator = + ThreadLocal.withInitial( + () -> new RowCompactedSerializer(keyType).createSliceComparator()); + Comparator keySliceComparator = + (left, right) -> threadLocalComparator.get().compare(left, right); this.lookupStoreFactory = LookupStoreFactory.create( options, new CacheManager( options.lookupCacheMaxMemory(), options.lookupCacheHighPrioPoolRatio()), - new RowCompactedSerializer(keyType).createSliceComparator()); + keySliceComparator); startLevel = options.needLookup() ? 1 : 0; } diff --git a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java index 073520efa83..1ae0400ef14 100644 --- a/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java +++ b/fluss-lake/fluss-lake-paimon/src/test/java/org/apache/fluss/lake/paimon/lookup/PaimonLakeTableLookuperTest.java @@ -329,11 +329,9 @@ void testConcurrentFirstLookupsForDifferentPartitions() throws Exception { () -> { downloadsStarted.countDown(); try { - // Without Fluss-level serialization, both downloads reach this - // guard and continue together, exercising Paimon's shared mutable - // lookup-store comparator. With serialization, the short wait - // expires and the downloads proceed one at a time. - downloadsStarted.await(100, TimeUnit.MILLISECONDS); + // Both lookup files must be created concurrently. This also + // exercises comparisons in different files after their downloads. + assertThat(downloadsStarted.await(10, TimeUnit.SECONDS)).isTrue(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException(e);