From 99754d74606eb25ec964516f18c86e5a611ccb53 Mon Sep 17 00:00:00 2001 From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com> Date: Sat, 6 Jan 2024 20:09:54 +0800 Subject: [PATCH] [improve](routine-load) remove maximum limit of routine load max_batch_interval (#29071) --- .../Data-Manipulation-Statements/Load/CREATE-ROUTINE-LOAD.md | 2 +- .../Data-Manipulation-Statements/Load/CREATE-ROUTINE-LOAD.md | 2 +- .../java/org/apache/doris/analysis/AlterRoutineLoadStmt.java | 2 +- .../java/org/apache/doris/analysis/CreateRoutineLoadStmt.java | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/CREATE-ROUTINE-LOAD.md b/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/CREATE-ROUTINE-LOAD.md index 86e9b246f5..2d071a7e45 100644 --- a/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/CREATE-ROUTINE-LOAD.md +++ b/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/CREATE-ROUTINE-LOAD.md @@ -159,7 +159,7 @@ FROM data_source [data_source_properties] These three parameters represent: - 1. The maximum execution time of each subtask, in seconds. The range is 1 to 60. Default is 10. + 1. The maximum execution time of each subtask, in seconds. Must be greater than or equal to 1. The default is 10. 2. The maximum number of lines read by each subtask. Must be greater than or equal to 200000. The default is 200000. 3. The maximum number of bytes read by each subtask. The unit is bytes and the range is 100MB to 1GB. The default is 100MB. diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/CREATE-ROUTINE-LOAD.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/CREATE-ROUTINE-LOAD.md index a365b26ea3..d86944935e 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/CREATE-ROUTINE-LOAD.md +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/CREATE-ROUTINE-LOAD.md @@ -156,7 +156,7 @@ FROM data_source [data_source_properties] 这三个参数分别表示： - 1. 每个子任务最大执行时间，单位是秒。范围为 1 到 60。默认为10。 + 1. 每个子任务最大执行时间，单位是秒。必须大于等于 1。默认为10。 2. 每个子任务最多读取的行数。必须大于等于200000。默认是200000。 3. 每个子任务最多读取的字节数。单位是字节，范围是 100MB 到 1GB。默认是 100MB。 diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRoutineLoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRoutineLoadStmt.java index 23488c120a..eb339360d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRoutineLoadStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterRoutineLoadStmt.java @@ -181,7 +181,7 @@ public class AlterRoutineLoadStmt extends DdlStmt { long maxBatchIntervalS = Util.getLongPropertyOrDefault( jobProperties.get(CreateRoutineLoadStmt.MAX_BATCH_INTERVAL_SEC_PROPERTY), -1, CreateRoutineLoadStmt.MAX_BATCH_INTERVAL_PRED, - CreateRoutineLoadStmt.MAX_BATCH_INTERVAL_SEC_PROPERTY + " should between 1 and 60"); + CreateRoutineLoadStmt.MAX_BATCH_INTERVAL_SEC_PROPERTY + " should >= 1"); analyzedJobProperties.put(CreateRoutineLoadStmt.MAX_BATCH_INTERVAL_SEC_PROPERTY, String.valueOf(maxBatchIntervalS)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java index 9710fe76b7..ff0a9053c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRoutineLoadStmt.java @@ -194,7 +194,7 @@ public class CreateRoutineLoadStmt extends DdlStmt { public static final Predicate DESIRED_CONCURRENT_NUMBER_PRED = (v) -> v > 0L; public static final Predicate MAX_ERROR_NUMBER_PRED = (v) -> v >= 0L; public static final Predicate MAX_FILTER_RATIO_PRED = (v) -> v >= 0 && v <= 1; - public static final Predicate MAX_BATCH_INTERVAL_PRED = (v) -> v >= 1 && v <= 60; + public static final Predicate MAX_BATCH_INTERVAL_PRED = (v) -> v >= 1; public static final Predicate MAX_BATCH_ROWS_PRED = (v) -> v >= 200000; public static final Predicate MAX_BATCH_SIZE_PRED = (v) -> v >= 100 * 1024 * 1024 && v <= 1024 * 1024 * 1024; public static final Predicate EXEC_MEM_LIMIT_PRED = (v) -> v >= 0L; @@ -474,7 +474,7 @@ public class CreateRoutineLoadStmt extends DdlStmt { maxBatchIntervalS = Util.getLongPropertyOrDefault(jobProperties.get(MAX_BATCH_INTERVAL_SEC_PROPERTY), RoutineLoadJob.DEFAULT_MAX_INTERVAL_SECOND, MAX_BATCH_INTERVAL_PRED, - MAX_BATCH_INTERVAL_SEC_PROPERTY + " should between 1 and 60"); + MAX_BATCH_INTERVAL_SEC_PROPERTY + " should >= 1"); maxBatchRows = Util.getLongPropertyOrDefault(jobProperties.get(MAX_BATCH_ROWS_PROPERTY), RoutineLoadJob.DEFAULT_MAX_BATCH_ROWS, MAX_BATCH_ROWS_PRED,