From 2ba88ed2a80094b378003b942acf39ecde4fcffe Mon Sep 17 00:00:00 2001 From: walter Date: Fri, 8 Nov 2024 18:59:53 +0800 Subject: [PATCH] [improve](report) split agent batch tasks automaticlly #43257 (#43365) cherry pick from #43257 --- .../src/main/java/org/apache/doris/common/Config.java | 8 ++++++++ .../main/java/org/apache/doris/master/ReportHandler.java | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index afc2ceaedd..72ce287103 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1177,6 +1177,14 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true) public static int report_queue_size = 100; + // if the number of report task in FE exceed max_report_task_num_per_rpc, then split it to multiple rpc + @ConfField(mutable = true, masterOnly = true, description = { + "重新发送 agent task 时，单次 RPC 分配给每个be的任务最大个数，默认值为10000个。", + "The maximum number of batched tasks per RPC assigned to each BE when resending agent tasks, " + + "the default value is 10000." + }) + public static int report_resend_batch_task_num_per_rpc = 10000; + /** * If set to true, metric collector will be run as a daemon timer to collect metrics at fix interval */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java index 0e2b14ed3f..2cb0f41aa1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java @@ -599,7 +599,7 @@ public class ReportHandler extends Daemon { List diffTasks = AgentTaskQueue.getDiffTasks(backendId, runningTasks); - AgentBatchTask batchTask = new AgentBatchTask(); + AgentBatchTask batchTask = new AgentBatchTask(Config.report_resend_batch_task_num_per_rpc); long taskReportTime = System.currentTimeMillis(); for (AgentTask task : diffTasks) { // these tasks no need to do diff