database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](catalog) modify some meta cache logic (#38506) (#39628)

Browse Source 
#38506
This commit is contained in:
Mingyu Chen
2024-08-20 21:57:55 +08:00
committed by GitHub
parent 12ed2951c4
commit 0e21dba817
7 changed files with 22 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
View File

@ -236,7 +236,7 @@ public abstract class ExternalCatalog
name,
OptionalLong.of(86400L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
Config.max_hive_table_cache_num,
Config.max_meta_object_cache_num,
ignored -> getFilteredDatabaseNames(),
dbName -> Optional.ofNullable(
buildDbForInit(dbName, Util.genIdByName(name, dbName), logType)),
@ -667,8 +667,6 @@ public abstract class ExternalCatalog
return new IcebergExternalDatabase(this, dbId, dbName);
case MAX_COMPUTE:
return new MaxComputeExternalDatabase(this, dbId, dbName);
//case HUDI:
//return new HudiExternalDatabase(this, dbId, dbName);
case TEST:
return new TestExternalDatabase(this, dbId, dbName);
case PAIMON:

2
fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
View File

@ -145,7 +145,7 @@ public abstract class ExternalDatabase<T extends ExternalTable>
name,
OptionalLong.of(86400L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
Config.max_hive_table_cache_num,
Config.max_meta_object_cache_num,
ignored -> listTableNames(),
tableName -> Optional.ofNullable(
buildTableForInit(tableName,

2
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
View File

@ -179,7 +179,7 @@ public class HiveMetaStoreCache {
CacheFactory fileCacheFactory = new CacheFactory(
OptionalLong.of(fileMetaCacheTtlSecond >= HMSExternalCatalog.FILE_META_CACHE_TTL_DISABLE_CACHE
? fileMetaCacheTtlSecond : 86400L),
? fileMetaCacheTtlSecond : 28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
Config.max_external_file_cache_num,
false,

4
fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java
View File

@ -52,9 +52,9 @@ public class HudiCachedPartitionProcessor extends HudiPartitionProcessor {
this.catalogId = catalogId;
this.executor = executor;
CacheFactory partitionCacheFactory = new CacheFactory(
OptionalLong.of(86400L),
OptionalLong.of(28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60),
Config.max_hive_table_cache_num,
Config.max_external_table_cache_num,
false,
null);
this.partitionCache = partitionCacheFactory.buildCache(key -> new TablePartitionValues(), null, executor);

8
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java
View File

@ -55,17 +55,17 @@ public class IcebergMetadataCache {
public IcebergMetadataCache(ExecutorService executor) {
CacheFactory snapshotListCacheFactory = new CacheFactory(
OptionalLong.of(86400L),
OptionalLong.of(28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60),
Config.max_hive_table_cache_num,
Config.max_external_table_cache_num,
false,
null);
this.snapshotListCache = snapshotListCacheFactory.buildCache(key -> loadSnapshots(key), null, executor);
CacheFactory tableCacheFactory = new CacheFactory(
OptionalLong.of(86400L),
OptionalLong.of(28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60),
Config.max_hive_table_cache_num,
Config.max_external_table_cache_num,
false,
null);
this.tableCache = tableCacheFactory.buildCache(key -> loadTable(key), null, executor);

2
fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java
View File

@ -57,7 +57,7 @@ public class MetaCache<T> {
CacheFactory namesCacheFactory = new CacheFactory(
expireAfterWriteSec,
refreshAfterWriteSec,
maxSize,
1, // names cache has one and only one entry
true,
null);
CacheFactory objCacheFactory = new CacheFactory(