database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](routine-load) optimize allocate task to be algorithm for load balance (#32021)

Browse Source
This commit is contained in:
HHoflittlefish777
2024-03-20 16:34:28 +08:00
committed by yiguolei
parent 6871c964af
commit 22782fec92
2 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java
View File

@ -484,7 +484,7 @@ public class RoutineLoadManager implements Writable {
try {
Map<Long, Integer> beIdToConcurrentTasks = getBeCurrentTasksNumMap();
// 1. Find if the given BE id has available slots
// 1. Find if the given BE id has more than half of available slots
if (previousBeId != -1L && availableBeIds.contains(previousBeId)) {
// get the previousBackend info
Backend previousBackend = Env.getCurrentSystemInfo().getBackend(previousBeId);
@ -499,7 +499,7 @@ public class RoutineLoadManager implements Writable {
} else {
idleTaskNum = beIdToMaxConcurrentTasks.get(previousBeId);
}
if (idleTaskNum > 0) {
if (idleTaskNum > (Config.max_routine_load_task_num_per_be >> 1)) {
return previousBeId;
}
}

3
fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java
View File

@ -299,7 +299,8 @@ public class RoutineLoadTaskScheduler extends MasterDaemon {
}
// try to allocate a task to BE which has idle slot.
// 1. First is to check if the previous allocated BE is available. If yes, allocate task to previous BE.
// 1. First is to check if the previous allocated BE has more than half of available slots.
// If yes, allocate task to previous BE.
// 2. If not, try to find a better one with most idle slots.
// return true if allocate successfully. return false if failed.
// throw exception if unrecoverable errors happen.