From e143dd2c1a2952de935e33aa86b62efbec4152e2 Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Wed, 19 Jun 2019 19:41:48 +0800 Subject: [PATCH] Modify the max keep time of historical alter jobs (#1334) Currently, historical alter jobs will keep for a while before being removed. And this time is configured by label_keep_max_second. Which is also used for Load jobs. But to avoid too many historical load jobs being kept in memory, 'label_keep_max_second' always set to a short time, causing alter jobs to be removed vary soon. Add a new FE config 'history_job_keep_max_second' to configure the keep time of alter jobs. Default is 7 days. --- fe/src/main/java/org/apache/doris/alter/AlterHandler.java | 2 +- fe/src/main/java/org/apache/doris/alter/RollupJob.java | 4 ++-- fe/src/main/java/org/apache/doris/catalog/Catalog.java | 2 +- fe/src/main/java/org/apache/doris/common/Config.java | 8 +++++++- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java index 6bff6c45d4..06849f41e6 100644 --- a/fe/src/main/java/org/apache/doris/alter/AlterHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/AlterHandler.java @@ -256,7 +256,7 @@ public abstract class AlterHandler extends Daemon { Iterator iter = finishedOrCancelledAlterJobs.iterator(); while (iter.hasNext()) { AlterJob historyJob = iter.next(); - if ((System.currentTimeMillis() - historyJob.getCreateTimeMs()) / 1000 > Config.label_keep_max_second) { + if ((System.currentTimeMillis() - historyJob.getCreateTimeMs()) / 1000 > Config.history_job_keep_max_second) { iter.remove(); LOG.info("remove history {} job[{}]. created at {}", historyJob.getType(), historyJob.getTableId(), TimeUtils.longToTimeString(historyJob.getCreateTimeMs())); diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJob.java b/fe/src/main/java/org/apache/doris/alter/RollupJob.java index 8bc2a08fb6..bd39a219e7 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJob.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJob.java @@ -598,8 +598,8 @@ public class RollupJob extends AlterJob { setReplicaFinished(partitionId, rollupReplicaId); rollupReplica.setState(ReplicaState.NORMAL); - LOG.info("finished rollup replica[{}]. index[{}]. tablet[{}]. backend[{}]", - rollupReplicaId, rollupIndexId, rollupTabletId, task.getBackendId()); + LOG.info("finished rollup replica[{}]. index[{}]. tablet[{}]. backend[{}], version: {}-{}", + rollupReplicaId, rollupIndexId, rollupTabletId, task.getBackendId(), version, versionHash); } /* diff --git a/fe/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/src/main/java/org/apache/doris/catalog/Catalog.java index f26e0da593..b04a189fc9 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/src/main/java/org/apache/doris/catalog/Catalog.java @@ -1613,7 +1613,7 @@ public class Catalog { long tableId = dis.readLong(); newChecksum ^= tableId; AlterJob job = AlterJob.read(dis); - if ((currentTimeMs - job.getCreateTimeMs()) / 1000 <= Config.label_keep_max_second) { + if ((currentTimeMs - job.getCreateTimeMs()) / 1000 <= Config.history_job_keep_max_second) { // delete history jobs finishedOrCancelledAlterJobs.add(job); } diff --git a/fe/src/main/java/org/apache/doris/common/Config.java b/fe/src/main/java/org/apache/doris/common/Config.java index 40d023212a..0a7388b119 100644 --- a/fe/src/main/java/org/apache/doris/common/Config.java +++ b/fe/src/main/java/org/apache/doris/common/Config.java @@ -87,7 +87,13 @@ public class Config extends ConfigBase { * (Because all load jobs' info is kept in memory before being removed) */ @ConfField(mutable = true, masterOnly = true) - public static int label_keep_max_second = 7 * 24 * 3600; // 7 days + public static int label_keep_max_second = 3 * 24 * 3600; // 3 days + /* + * The max keep time of some kind of jobs. + * like schema change job and rollup job. + */ + @ConfField(mutable = true, masterOnly = true) + public static int history_job_keep_max_second = 7 * 24 * 3600; // 7 days /* * Load label cleaner will run every *label_clean_interval_second* to clean the outdated jobs. */