From 1964a77d6c7694dc956ffe0d4a73b61205f3ccaf Mon Sep 17 00:00:00 2001 From: zhengyu Date: Tue, 26 Dec 2023 16:22:52 +0800 Subject: [PATCH] [enhencement](config) change default memtable size & loadStreamPerNode & default load parallelism (#28977) We change memtable size from 200MB to 100MB to achieve smoother flush performance. We change loadStreamPerNode from 20 to 60 to avoid stream rpc to be the bottleneck when enable memtable_on_sink_node. We change default s3&broker load parallelsim to make the most of CPUs on moderm multi-core systems. Signed-off-by: freemandealer --- be/src/common/config.cpp | 2 +- be/src/common/config.h | 2 +- docs/en/docs/admin-manual/config/fe-config.md | 2 +- docs/zh-CN/docs/admin-manual/config/fe-config.md | 2 +- fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 2 +- .../src/main/java/org/apache/doris/qe/SessionVariable.java | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 1d8178ceec..7dc1ec680a 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -592,7 +592,7 @@ DEFINE_Int32(memory_max_alignment, "16"); // memtable insert memory tracker will multiply input block size with this ratio DEFINE_mDouble(memtable_insert_memory_ratio, "1.4"); // max write buffer size before flush, default 200MB -DEFINE_mInt64(write_buffer_size, "209715200"); +DEFINE_mInt64(write_buffer_size, "104857600"); // max buffer size used in memtable for the aggregated table, default 400MB DEFINE_mInt64(write_buffer_size_for_agg, "419430400"); // max parallel flush task per memtable writer diff --git a/be/src/common/config.h b/be/src/common/config.h index affe47851f..bb34100290 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -642,7 +642,7 @@ DECLARE_Int32(memory_max_alignment); // memtable insert memory tracker will multiply input block size with this ratio DECLARE_mDouble(memtable_insert_memory_ratio); -// max write buffer size before flush, default 200MB +// max write buffer size before flush, default 100MB DECLARE_mInt64(write_buffer_size); // max buffer size used in memtable for the aggregated table, default 400MB DECLARE_mInt64(write_buffer_size_for_agg); diff --git a/docs/en/docs/admin-manual/config/fe-config.md b/docs/en/docs/admin-manual/config/fe-config.md index 3721a364d5..ae3c0645be 100644 --- a/docs/en/docs/admin-manual/config/fe-config.md +++ b/docs/en/docs/admin-manual/config/fe-config.md @@ -1153,7 +1153,7 @@ Max bytes a broker scanner can process in one broker load job. Commonly, each Ba #### `default_load_parallelism` -Default: 1 +Default: 8 IsMutable：true diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md index 356677ae04..580cfe7fee 100644 --- a/docs/zh-CN/docs/admin-manual/config/fe-config.md +++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md @@ -1154,7 +1154,7 @@ broker scanner 程序可以在一个 broker 加载作业中处理的最大字节 #### `default_load_parallelism` -默认值：1 +默认值：8 是否可以动态配置：true diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index c038a9f178..61a135ed96 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -144,7 +144,7 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true, description = {"broker load 时，单个节点上 load 执行计划的默认并行度", "The default parallelism of the load execution plan on a single node when the broker load is submitted"}) - public static int default_load_parallelism = 1; + public static int default_load_parallelism = 8; @ConfField(mutable = true, masterOnly = true, description = { "已完成或取消的导入作业信息的 label 会在这个时间后被删除。被删除的 label 可以被重用。", diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 332e0f5bfa..e33a619367 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -1325,7 +1325,7 @@ public class SessionVariable implements Serializable, Writable { public boolean enableMemtableOnSinkNode = true; @VariableMgr.VarAttr(name = LOAD_STREAM_PER_NODE) - public int loadStreamPerNode = 20; + public int loadStreamPerNode = 60; @VariableMgr.VarAttr(name = GROUP_COMMIT) public String groupCommit = "off_mode";