database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](memory) reduce memory usage for failed broker loads (#15895)

Browse Source 
* [enhancement](memory) reduce memory usage for failed  broker loads
This commit is contained in:
Zhengguo Yang
2023-01-30 10:22:31 +08:00
committed by GitHub
parent 69e748b076
commit ec4a56922f
5 changed files with 29 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
View File

@ -24,6 +24,7 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DataQualityException;
import org.apache.doris.common.DuplicatedRequestException;
import org.apache.doris.common.LabelAlreadyUsedException;
@ -59,6 +60,7 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.RejectedExecutionException;
import java.util.stream.Collectors;
/**
* There are 3 steps in BrokerLoadJob: BrokerPendingTask, LoadLoadingTask, CommitAndPublishTxn.
@ -351,7 +353,8 @@ public class BrokerLoadJob extends BulkLoadJob {
loadingStatus.setTrackingUrl(attachment.getTrackingUrl());
}
commitInfos.addAll(attachment.getCommitInfoList());
errorTabletInfos.addAll(attachment.getErrorTabletInfos());
errorTabletInfos.addAll(attachment.getErrorTabletInfos().stream().limit(Config.max_error_tablet_of_broker_load)
.collect(Collectors.toList()));
progress = (int) ((double) finishedTaskIds.size() / idToTasks.size() * 100);
if (progress == 100) {

3
fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
View File

@ -797,7 +797,8 @@ public abstract class LoadJob extends AbstractTxnStateChangeCallback implements
public String errorTabletsToJson() {
Map<Long, String> map = Maps.newHashMap();
errorTabletInfos.stream().limit(3).forEach(p -> map.put(p.getTabletId(), p.getMsg()));
errorTabletInfos.stream().limit(Config.max_error_tablet_of_broker_load)
.forEach(p -> map.put(p.getTabletId(), p.getMsg()));
Gson gson = new GsonBuilder().disableHtmlEscaping().create();
return gson.toJson(map);
}