From 6bd26092945e91ab2c9f9f6367de4ec3d243c45d Mon Sep 17 00:00:00 2001 From: Xiangyu Wang Date: Tue, 28 Mar 2023 15:10:19 +0800 Subject: [PATCH] =?UTF-8?q?[Enhancement](multi-catalog)=20add=20config=20f?= =?UTF-8?q?or=20external=20meta=20cache=20loade=E2=80=A6=20(#18117)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add config for external cache-loader's max thread-pool size. --- docs/en/docs/admin-manual/config/fe-config.md | 12 +++++++++++- docs/zh-CN/docs/admin-manual/config/fe-config.md | 10 ++++++++++ .../main/java/org/apache/doris/common/Config.java | 7 +++++++ .../doris/datasource/ExternalMetaCacheMgr.java | 4 +++- 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/en/docs/admin-manual/config/fe-config.md b/docs/en/docs/admin-manual/config/fe-config.md index 8d3bba4deb..ea205c8b94 100644 --- a/docs/en/docs/admin-manual/config/fe-config.md +++ b/docs/en/docs/admin-manual/config/fe-config.md @@ -2328,9 +2328,19 @@ Is it possible to dynamically configure: true Is it a configuration item unique to the Master FE node: true +#### `max_external_cache_loader_thread_pool_size` + +Maximum thread pool size for loading external meta cache. + +Default: 10 + +Is it possible to dynamically configure: false + +Is it a configuration item unique to the Master FE node: false + #### `max_external_file_cache_num` -Maximum number of file cache to use for external external tables. +Maximum number of file cache to use for external tables. Default: 100000 diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md index 389b3fb8a7..3de2adb966 100644 --- a/docs/zh-CN/docs/admin-manual/config/fe-config.md +++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md @@ -2328,6 +2328,16 @@ hive metastore 的默认超时时间 是否为 Master FE 节点独有的配置项：true +#### `max_external_cache_loader_thread_pool_size` + +用于 external 外部表的 meta 缓存加载线程池的最大线程数。 + +默认值：10 + +是否可以动态配置：false + +是否为 Master FE 节点独有的配置项：false + #### `max_external_file_cache_num` 用于 external 外部表的最大文件缓存数量。 diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 551de31252..92ad5aa798 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1904,6 +1904,13 @@ public class Config extends ConfigBase { @ConfField(mutable = false, masterOnly = false) public static long max_hive_partition_cache_num = 100000; + /** + * Max cache loader thread-pool size. + * Max thread pool size for loading external meta cache + */ + @ConfField(mutable = false, masterOnly = false) + public static int max_external_cache_loader_thread_pool_size = 10; + /** * Max cache num of external catalog's file * Decrease this value if FE's memory is small diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index 13875d084e..afe1cce61c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -20,6 +20,7 @@ package org.apache.doris.datasource; import org.apache.doris.catalog.external.ExternalTable; import org.apache.doris.catalog.external.HMSExternalTable; import org.apache.doris.cluster.ClusterNamespace; +import org.apache.doris.common.Config; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.datasource.hive.HiveMetaStoreCache; @@ -46,7 +47,8 @@ public class ExternalMetaCacheMgr { private Executor executor; public ExternalMetaCacheMgr() { - executor = ThreadPoolManager.newDaemonCacheThreadPool(10, "ExternalMetaCacheMgr", false); + executor = ThreadPoolManager.newDaemonCacheThreadPool(Config.max_external_cache_loader_thread_pool_size, + "ExternalMetaCacheMgr", true); } public HiveMetaStoreCache getMetaStoreCache(HMSExternalCatalog catalog) {