database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Modify the max keep time of historical alter jobs (#1334)

Browse Source 
Currently, historical alter jobs will keep for a while before being removed.
And this time is configured by label_keep_max_second. Which is also used for
Load jobs.

But to avoid too many historical load jobs being kept in memory,
'label_keep_max_second' always set to a short time, causing alter jobs to be
removed vary soon.

Add a new FE config 'history_job_keep_max_second' to configure the keep time of
alter jobs. Default is 7 days.
This commit is contained in:
Mingyu Chen
2019-06-19 19:41:48 +08:00
committed by GitHub
parent 5cd4777bb4
commit e143dd2c1a
4 changed files with 11 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/src/main/java/org/apache/doris/alter/AlterHandler.java
View File

@ -256,7 +256,7 @@ public abstract class AlterHandler extends Daemon {
Iterator<AlterJob> iter = finishedOrCancelledAlterJobs.iterator();
while (iter.hasNext()) {
AlterJob historyJob = iter.next();
if ((System.currentTimeMillis() - historyJob.getCreateTimeMs()) / 1000 > Config.label_keep_max_second) {
if ((System.currentTimeMillis() - historyJob.getCreateTimeMs()) / 1000 > Config.history_job_keep_max_second) {
iter.remove();
LOG.info("remove history {} job[{}]. created at {}", historyJob.getType(),
historyJob.getTableId(), TimeUtils.longToTimeString(historyJob.getCreateTimeMs()));

4
fe/src/main/java/org/apache/doris/alter/RollupJob.java
View File

@ -598,8 +598,8 @@ public class RollupJob extends AlterJob {
setReplicaFinished(partitionId, rollupReplicaId);
rollupReplica.setState(ReplicaState.NORMAL);
LOG.info("finished rollup replica[{}]. index[{}]. tablet[{}]. backend[{}]",
rollupReplicaId, rollupIndexId, rollupTabletId, task.getBackendId());
LOG.info("finished rollup replica[{}]. index[{}]. tablet[{}]. backend[{}], version: {}-{}",
rollupReplicaId, rollupIndexId, rollupTabletId, task.getBackendId(), version, versionHash);
}
/*

2
fe/src/main/java/org/apache/doris/catalog/Catalog.java
View File

@ -1613,7 +1613,7 @@ public class Catalog {
long tableId = dis.readLong();
newChecksum ^= tableId;
AlterJob job = AlterJob.read(dis);
if ((currentTimeMs - job.getCreateTimeMs()) / 1000 <= Config.label_keep_max_second) {
if ((currentTimeMs - job.getCreateTimeMs()) / 1000 <= Config.history_job_keep_max_second) {
// delete history jobs
finishedOrCancelledAlterJobs.add(job);
}

8
fe/src/main/java/org/apache/doris/common/Config.java
View File

@ -87,7 +87,13 @@ public class Config extends ConfigBase {
* (Because all load jobs' info is kept in memory before being removed)
*/
@ConfField(mutable = true, masterOnly = true)
public static int label_keep_max_second = 7 * 24 * 3600; // 7 days
public static int label_keep_max_second = 3 * 24 * 3600; // 3 days
/*
* The max keep time of some kind of jobs.
* like schema change job and rollup job.
*/
@ConfField(mutable = true, masterOnly = true)
public static int history_job_keep_max_second = 7 * 24 * 3600; // 7 days
/*
* Load label cleaner will run every *label_clean_interval_second* to clean the outdated jobs.
*/