database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improve](routine-load) increase routing load max_batch _size max limit (#31846)

Browse Source
This commit is contained in:
HHoflittlefish777
2024-03-08 12:42:16 +08:00
committed by yiguolei
parent 19e6ebd09c
commit e3611f6a1d
5 changed files with 7 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRoutineLoadStmt.java
View File

@ -199,7 +199,7 @@ public class AlterRoutineLoadStmt extends DdlStmt {
long maxBatchSizeBytes = Util.getLongPropertyOrDefault(
jobProperties.get(CreateRoutineLoadStmt.MAX_BATCH_SIZE_PROPERTY),
-1, CreateRoutineLoadStmt.MAX_BATCH_SIZE_PRED,
CreateRoutineLoadStmt.MAX_BATCH_SIZE_PROPERTY + " should between 100MB and 1GB");
CreateRoutineLoadStmt.MAX_BATCH_SIZE_PROPERTY + " should between 100MB and 10GB");
analyzedJobProperties.put(CreateRoutineLoadStmt.MAX_BATCH_SIZE_PROPERTY,
String.valueOf(maxBatchSizeBytes));
}

5
fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java
View File

@ -196,7 +196,8 @@ public class CreateRoutineLoadStmt extends DdlStmt {
public static final Predicate<Double> MAX_FILTER_RATIO_PRED = (v) -> v >= 0 && v <= 1;
public static final Predicate<Long> MAX_BATCH_INTERVAL_PRED = (v) -> v >= 1;
public static final Predicate<Long> MAX_BATCH_ROWS_PRED = (v) -> v >= 200000;
public static final Predicate<Long> MAX_BATCH_SIZE_PRED = (v) -> v >= 100 * 1024 * 1024 && v <= 1024 * 1024 * 1024;
public static final Predicate<Long> MAX_BATCH_SIZE_PRED = (v) -> v >= 100 * 1024 * 1024
&& v <= (long) (1024 * 1024 * 1024) * 10;
public static final Predicate<Long> EXEC_MEM_LIMIT_PRED = (v) -> v >= 0L;
public static final Predicate<Long> SEND_BATCH_PARALLELISM_PRED = (v) -> v > 0L;
@ -482,7 +483,7 @@ public class CreateRoutineLoadStmt extends DdlStmt {
maxBatchSizeBytes = Util.getLongPropertyOrDefault(jobProperties.get(MAX_BATCH_SIZE_PROPERTY),
RoutineLoadJob.DEFAULT_MAX_BATCH_SIZE, MAX_BATCH_SIZE_PRED,
MAX_BATCH_SIZE_PROPERTY + " should between 100MB and 1GB");
MAX_BATCH_SIZE_PROPERTY + " should between 100MB and 10GB");
strictMode = Util.getBooleanPropertyOrDefault(jobProperties.get(LoadStmt.STRICT_MODE),
RoutineLoadJob.DEFAULT_STRICT_MODE,