database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](memory) reduce memory usage for failed broker loads (#16974)

Browse Source 
Reduce more memory usage for failed broker load msg in fe after pr  #15895
This commit is contained in:
Zhengguo Yang
2023-02-24 12:07:02 +08:00
committed by GitHub
parent 03a4fe6f39
commit d562428b1d
2 changed files with 8 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
View File

@ -45,6 +45,7 @@ import org.apache.logging.log4j.Logger;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;
public class LoadLoadingTask extends LoadTask {
private static final Logger LOG = LogManager.getLogger(LoadLoadingTask.class);
@ -177,7 +178,9 @@ public class LoadLoadingTask extends LoadTask {
curCoordinator.getLoadCounters(),
curCoordinator.getTrackingUrl(),
TabletCommitInfo.fromThrift(curCoordinator.getCommitInfos()),
ErrorTabletInfo.fromThrift(curCoordinator.getErrorTabletInfos()));
ErrorTabletInfo.fromThrift(curCoordinator.getErrorTabletInfos()
.stream().limit(Config.max_error_tablet_of_broker_load).collect(Collectors.toList())));
curCoordinator.getErrorTabletInfos().clear();
// Create profile of this task and add to the job profile.
createProfile(curCoordinator);
} else {

5
fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
View File

@ -1125,7 +1125,10 @@ public class Coordinator {
private void updateErrorTabletInfos(List<TErrorTabletInfo> errorTabletInfos) {
lock.lock();
try {
this.errorTabletInfos.addAll(errorTabletInfos);
if (this.errorTabletInfos.size() <= Config.max_error_tablet_of_broker_load) {
this.errorTabletInfos.addAll(errorTabletInfos.stream().limit(Config.max_error_tablet_of_broker_load
- this.errorTabletInfos.size()).collect(Collectors.toList()));
}
} finally {
lock.unlock();
}