database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](fe) Reduce jvm heap memory consumed by profiles of BrokerLoadJob (#31985)

Browse Source 
* it may cause FE OOM when there are a lot of broker load jobs
  if the profile is enabled
This commit is contained in:
Lei Zhang
2024-03-08 19:33:31 +08:00
committed by yiguolei
parent 21e412393b
commit 4bdea7c324
1 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
View File

@ -198,7 +198,9 @@ public class BrokerLoadJob extends BulkLoadJob {
Lists.newArrayList(fileGroupAggInfo.getAllTableIds()));
// divide job into broker loading task by table
List<LoadLoadingTask> newLoadingTasks = Lists.newArrayList();
this.jobProfile = new Profile("BrokerLoadJob " + id + ". " + label, true);
if (enableProfile) {
this.jobProfile = new Profile("BrokerLoadJob " + id + ". " + label, true);
}
ProgressManager progressManager = Env.getCurrentProgressManager();
progressManager.registerProgressSimple(String.valueOf(id));
MetaLockUtils.readLockTables(tableList);
@ -333,6 +335,8 @@ public class BrokerLoadJob extends BulkLoadJob {
}
jobProfile.update(createTimestamp, getSummaryInfo(true), true,
Integer.valueOf(sessionVariables.getOrDefault(SessionVariable.PROFILE_LEVEL, "3")), null, false);
// jobProfile has been pushed into ProfileManager, remove reference in brokerLoadJob
jobProfile = null;
}
private Map<String, String> getSummaryInfo(boolean isFinished) {