From e670d84b727362258e067fbe4bf4ce49122c32ef Mon Sep 17 00:00:00 2001
From: Mryange <59914473+Mryange@users.noreply.github.com>
Date: Thu, 3 Aug 2023 13:12:32 +0800
Subject: [PATCH] [feature](executor) using max_instance_num to limit
automatically instance (#22521)
---
docs/en/docs/admin-manual/config/fe-config.md | 9 ---------
.../pipeline-execution-engine.md | 1 +
.../docs/admin-manual/config/fe-config.md | 8 --------
.../pipeline-execution-engine.md | 2 +-
.../java/org/apache/doris/common/Config.java | 7 -------
.../java/org/apache/doris/analysis/SetVar.java | 18 +-----------------
.../org/apache/doris/qe/SessionVariable.java | 7 ++++++-
7 files changed, 9 insertions(+), 43 deletions(-)
diff --git a/docs/en/docs/admin-manual/config/fe-config.md b/docs/en/docs/admin-manual/config/fe-config.md
index 0477f43813..a1ed75684c 100644
--- a/docs/en/docs/admin-manual/config/fe-config.md
+++ b/docs/en/docs/admin-manual/config/fe-config.md
@@ -2732,15 +2732,6 @@ MasterOnly: false
Controls whether to enable query hit statistics. The default is false.
-#### `max_instance_num`
-
-
-
-Default: 128
-
-This is used to limit the setting of "parallel_fragment_exec_instance_num".
-"parallel_fragment_exec_instance_num" cannot be set higher than "max_instance_num".
-
#### `div_precision_increment`
diff --git a/docs/en/docs/query-acceleration/pipeline-execution-engine.md b/docs/en/docs/query-acceleration/pipeline-execution-engine.md
index d32ca42217..8d3a2697c1 100644
--- a/docs/en/docs/query-acceleration/pipeline-execution-engine.md
+++ b/docs/en/docs/query-acceleration/pipeline-execution-engine.md
@@ -77,3 +77,4 @@ If the user upgrades from a lower version, the default value will be the paralle
```
set parallel_pipeline_task_num = 0;
```
+You can limit the automatically configured concurrency by setting "max_instance_num."（The default value is 64)
diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md
index d404f2cf4e..8eb3436939 100644
--- a/docs/zh-CN/docs/admin-manual/config/fe-config.md
+++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md
@@ -2724,14 +2724,6 @@ show data （其他用法：HELP SHOW DATA）
这个参数主要用于避免因 external catalog 无法访问、信息过多等原因导致的查询 `information_schema` 超时的问题。
-#### `max_instance_num`
-
-
-
-默认值：128
-
-用于限制parallel_fragment_exec_instance_num的设置，set parallel_fragment_exec_instance_num不能超过max_instance_num
-
#### `enable_query_hit_stats`
diff --git a/docs/zh-CN/docs/query-acceleration/pipeline-execution-engine.md b/docs/zh-CN/docs/query-acceleration/pipeline-execution-engine.md
index 0ce0f65448..3953d9c8d8 100644
--- a/docs/zh-CN/docs/query-acceleration/pipeline-execution-engine.md
+++ b/docs/zh-CN/docs/query-acceleration/pipeline-execution-engine.md
@@ -77,4 +77,4 @@ set enable_pipeline_engine = true;
```
set parallel_pipeline_task_num = 0;
```
-
+可以通过设置max_instance_num来限制自动设置的并发数(默认为64)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 730aa08680..11c85db137 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1961,13 +1961,6 @@ public class Config extends ConfigBase {
"Now default set to true, not support create complex type(array/struct/map) nested complex type "
+ "when we create table, only support array type nested array"})
public static boolean disable_nested_complex_type = true;
- /*
- * "max_instance_num" is used to set the maximum concurrency. When the value set
- * by "parallel_fragment_exec_instance_num" is greater than "max_instance_num",
- * an error will be reported.
- */
- @ConfField(mutable = true)
- public static int max_instance_num = 128;
/*
* This variable indicates the number of digits by which to increase the scale
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetVar.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetVar.java
index a0d61248a5..6142feec89 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetVar.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetVar.java
@@ -19,7 +19,6 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
@@ -179,22 +178,7 @@ public class SetVar {
this.value = new StringLiteral(TimeUtils.checkTimeZoneValidAndStandardize(getResult().getStringValue()));
this.result = (LiteralExpr) this.value;
}
- if (getVariable().equalsIgnoreCase(SessionVariable.PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM)) {
- int instanceNum = Integer.parseInt(getResult().getStringValue());
- if (instanceNum > Config.max_instance_num) {
- ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_VALUE_FOR_VAR,
- SessionVariable.PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM,
- instanceNum + "(Should not be set to more than " + Config.max_instance_num + ")");
- }
- }
- if (getVariable().equalsIgnoreCase(SessionVariable.PARALLEL_PIPELINE_TASK_NUM)) {
- int instanceNum = Integer.parseInt(getValue().getStringValue());
- if (instanceNum > Config.max_instance_num) {
- ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_VALUE_FOR_VAR,
- SessionVariable.PARALLEL_PIPELINE_TASK_NUM,
- instanceNum + "(Should not be set to more than " + Config.max_instance_num + ")");
- }
- }
+
if (getVariable().equalsIgnoreCase(SessionVariable.EXEC_MEM_LIMIT)) {
this.value = new StringLiteral(Long.toString(ParseUtil.analyzeDataVolumn(getResult().getStringValue())));
this.result = (LiteralExpr) this.value;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 9e87681d3d..b2b9e9a7a4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -106,6 +106,7 @@ public class SessionVariable implements Serializable, Writable {
public static final String ENABLE_BUCKET_SHUFFLE_JOIN = "enable_bucket_shuffle_join";
public static final String PARALLEL_FRAGMENT_EXEC_INSTANCE_NUM = "parallel_fragment_exec_instance_num";
public static final String PARALLEL_PIPELINE_TASK_NUM = "parallel_pipeline_task_num";
+ public static final String MAX_INSTANCE_NUM = "max_instance_num";
public static final String ENABLE_INSERT_STRICT = "enable_insert_strict";
public static final String ENABLE_SPILLING = "enable_spilling";
public static final String ENABLE_EXCHANGE_NODE_PARALLEL_MERGE = "enable_exchange_node_parallel_merge";
@@ -566,6 +567,9 @@ public class SessionVariable implements Serializable, Writable {
@VariableMgr.VarAttr(name = PARALLEL_PIPELINE_TASK_NUM, fuzzy = true)
public int parallelPipelineTaskNum = 0;
+ @VariableMgr.VarAttr(name = MAX_INSTANCE_NUM)
+ public int maxInstanceNum = 64;
+
@VariableMgr.VarAttr(name = ENABLE_INSERT_STRICT, needForward = true)
public boolean enableInsertStrict = true;
@@ -1476,7 +1480,8 @@ public class SessionVariable implements Serializable, Writable {
public int getParallelExecInstanceNum() {
if (enablePipelineEngine && parallelPipelineTaskNum == 0) {
int size = Env.getCurrentSystemInfo().getMinPipelineExecutorSize();
- return (size + 1) / 2;
+ int autoInstance = (size + 1) / 2;
+ return Math.min(autoInstance, maxInstanceNum);
} else if (enablePipelineEngine) {
return parallelPipelineTaskNum;
} else {