From 0e21dba817180bcfb491bcf8aff81273c5af45b2 Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Tue, 20 Aug 2024 21:57:55 +0800 Subject: [PATCH] [opt](catalog) modify some meta cache logic (#38506) (#39628) #38506 --- .../java/org/apache/doris/common/Config.java | 20 +++++++++++-------- .../doris/datasource/ExternalCatalog.java | 4 +--- .../doris/datasource/ExternalDatabase.java | 2 +- .../datasource/hive/HiveMetaStoreCache.java | 2 +- .../source/HudiCachedPartitionProcessor.java | 4 ++-- .../iceberg/IcebergMetadataCache.java | 8 ++++---- .../doris/datasource/metacache/MetaCache.java | 2 +- 7 files changed, 22 insertions(+), 20 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 602e5dffdb..35fdac5968 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1968,16 +1968,20 @@ public class Config extends ConfigBase { * Max cache num of hive partition. * Decrease this value if FE's memory is small */ - @ConfField(mutable = false, masterOnly = false) - public static long max_hive_partition_cache_num = 100000; + @ConfField(description = {"Hive Metastore 表级别分区缓存的最大数量。", + "Max cache number of partition at table level in Hive Metastore."}) + public static long max_hive_partition_cache_num = 10000; - @ConfField(mutable = false, masterOnly = false, description = {"Hive表名缓存的最大数量。", - "Max cache number of hive table name list."}) - public static long max_hive_table_cache_num = 1000; + @ConfField(description = {"Hudi/Iceberg 表级别缓存的最大数量。", + "Max cache number of hudi/iceberg table."}) + public static long max_external_table_cache_num = 1000; - @ConfField(mutable = false, masterOnly = false, description = { - "Hive分区表缓存的最大数量", "Max cache number of hive partition table" - }) + @ConfField(description = {"External Catalog 中，Database 和 Table 的实例缓存的最大数量。", + "Max cache number of database and table instance in external catalog."}) + public static long max_meta_object_cache_num = 1000; + + @ConfField(description = {"Hive分区表缓存的最大数量", + "Max cache number of hive partition table"}) public static long max_hive_partition_table_cache_num = 1000; @ConfField(mutable = false, masterOnly = false, description = {"获取Hive分区值时候的最大返回数量，-1代表没有限制。", diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 7f8c0c71b5..974f052bb1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -236,7 +236,7 @@ public abstract class ExternalCatalog name, OptionalLong.of(86400L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), - Config.max_hive_table_cache_num, + Config.max_meta_object_cache_num, ignored -> getFilteredDatabaseNames(), dbName -> Optional.ofNullable( buildDbForInit(dbName, Util.genIdByName(name, dbName), logType)), @@ -667,8 +667,6 @@ public abstract class ExternalCatalog return new IcebergExternalDatabase(this, dbId, dbName); case MAX_COMPUTE: return new MaxComputeExternalDatabase(this, dbId, dbName); - //case HUDI: - //return new HudiExternalDatabase(this, dbId, dbName); case TEST: return new TestExternalDatabase(this, dbId, dbName); case PAIMON: diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index b564a17ce8..dc6f9aaea7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -145,7 +145,7 @@ public abstract class ExternalDatabase name, OptionalLong.of(86400L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), - Config.max_hive_table_cache_num, + Config.max_meta_object_cache_num, ignored -> listTableNames(), tableName -> Optional.ofNullable( buildTableForInit(tableName, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index 81541fce49..312f2382b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -179,7 +179,7 @@ public class HiveMetaStoreCache { CacheFactory fileCacheFactory = new CacheFactory( OptionalLong.of(fileMetaCacheTtlSecond >= HMSExternalCatalog.FILE_META_CACHE_TTL_DISABLE_CACHE - ? fileMetaCacheTtlSecond : 86400L), + ? fileMetaCacheTtlSecond : 28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), Config.max_external_file_cache_num, false, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java index d9c1c20827..2372b88e0d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java @@ -52,9 +52,9 @@ public class HudiCachedPartitionProcessor extends HudiPartitionProcessor { this.catalogId = catalogId; this.executor = executor; CacheFactory partitionCacheFactory = new CacheFactory( - OptionalLong.of(86400L), + OptionalLong.of(28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60), - Config.max_hive_table_cache_num, + Config.max_external_table_cache_num, false, null); this.partitionCache = partitionCacheFactory.buildCache(key -> new TablePartitionValues(), null, executor); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java index dc11a6cacc..13bd965097 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java @@ -55,17 +55,17 @@ public class IcebergMetadataCache { public IcebergMetadataCache(ExecutorService executor) { CacheFactory snapshotListCacheFactory = new CacheFactory( - OptionalLong.of(86400L), + OptionalLong.of(28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60), - Config.max_hive_table_cache_num, + Config.max_external_table_cache_num, false, null); this.snapshotListCache = snapshotListCacheFactory.buildCache(key -> loadSnapshots(key), null, executor); CacheFactory tableCacheFactory = new CacheFactory( - OptionalLong.of(86400L), + OptionalLong.of(28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60), - Config.max_hive_table_cache_num, + Config.max_external_table_cache_num, false, null); this.tableCache = tableCacheFactory.buildCache(key -> loadTable(key), null, executor); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java index c251db3a5c..e3ad8668fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java @@ -57,7 +57,7 @@ public class MetaCache { CacheFactory namesCacheFactory = new CacheFactory( expireAfterWriteSec, refreshAfterWriteSec, - maxSize, + 1, // names cache has one and only one entry true, null); CacheFactory objCacheFactory = new CacheFactory(