From 4bdea7c3249be91724c880d1866e383d47a9c4c2 Mon Sep 17 00:00:00 2001 From: Lei Zhang <27994433+SWJTU-ZhangLei@users.noreply.github.com> Date: Fri, 8 Mar 2024 19:33:31 +0800 Subject: [PATCH] [opt](fe) Reduce jvm heap memory consumed by profiles of BrokerLoadJob (#31985) * it may cause FE OOM when there are a lot of broker load jobs if the profile is enabled --- .../java/org/apache/doris/load/loadv2/BrokerLoadJob.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java index 50e46fc383..27503d4cc7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java @@ -198,7 +198,9 @@ public class BrokerLoadJob extends BulkLoadJob { Lists.newArrayList(fileGroupAggInfo.getAllTableIds())); // divide job into broker loading task by table List newLoadingTasks = Lists.newArrayList(); - this.jobProfile = new Profile("BrokerLoadJob " + id + ". " + label, true); + if (enableProfile) { + this.jobProfile = new Profile("BrokerLoadJob " + id + ". " + label, true); + } ProgressManager progressManager = Env.getCurrentProgressManager(); progressManager.registerProgressSimple(String.valueOf(id)); MetaLockUtils.readLockTables(tableList); @@ -333,6 +335,8 @@ public class BrokerLoadJob extends BulkLoadJob { } jobProfile.update(createTimestamp, getSummaryInfo(true), true, Integer.valueOf(sessionVariables.getOrDefault(SessionVariable.PROFILE_LEVEL, "3")), null, false); + // jobProfile has been pushed into ProfileManager, remove reference in brokerLoadJob + jobProfile = null; } private Map getSummaryInfo(boolean isFinished) {