database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Bug] fix finished load jobs cost too much heap (#4993)

Browse Source 
Since the plan is retained in the task, if the task is not cleaned up, the memory usage will be too large caused Memory leak or OOM.
When load job finished, there is no need to hold the tasks which are the biggest memory consumers.
Fixed #4992
This commit is contained in:
qiye
2020-12-02 17:11:27 +08:00
committed by GitHub
parent af06adb57f
commit b4c1eabe3f
2 changed files with 7 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
View File

@ -659,6 +659,8 @@ public abstract class LoadJob extends AbstractTxnStateChangeCallback implements
if (MetricRepo.isInit) {
MetricRepo.COUNTER_LOAD_FINISHED.increase(1L);
}
// when load job finished, there is no need to hold the tasks which are the biggest memory consumers.
idToTasks.clear();
}
protected boolean checkDataQuality() {

6
fe/fe-core/src/test/java/org/apache/doris/load/loadv2/LoadJobTest.java
View File

@ -40,13 +40,13 @@ import org.apache.doris.transaction.TransactionState;
import com.google.common.collect.Maps;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import java.util.Map;
import mockit.Expectations;
import mockit.Injectable;
import mockit.Mocked;
public class LoadJobTest {
@ -187,18 +187,22 @@ public class LoadJobTest {
@Test
public void testUpdateStateToFinished(@Mocked MetricRepo metricRepo,
@Injectable LoadTask loadTask1,
@Mocked LongCounterMetric longCounterMetric) {
MetricRepo.COUNTER_LOAD_FINISHED = longCounterMetric;
LoadJob loadJob = new BrokerLoadJob();
loadJob.idToTasks.put(1L, loadTask1);
// TxnStateCallbackFactory factory = Catalog.getCurrentCatalog().getGlobalTransactionMgr().getCallbackFactory();
Catalog catalog = Catalog.getCurrentCatalog();
GlobalTransactionMgr mgr = new GlobalTransactionMgr(catalog);
Deencapsulation.setField(catalog, "globalTransactionMgr", mgr);
Assert.assertEquals(1, loadJob.idToTasks.size());
loadJob.updateState(JobState.FINISHED);
Assert.assertEquals(JobState.FINISHED, loadJob.getState());
Assert.assertNotEquals(-1, (long) Deencapsulation.getField(loadJob, "finishTimestamp"));
Assert.assertEquals(100, (int)Deencapsulation.getField(loadJob, "progress"));
Assert.assertEquals(0, loadJob.idToTasks.size());
}
}