database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Feature](Job)Jobs in the Finish state will be automatically deleted after three days. (#25170)

Browse Source
This commit is contained in:
Calvin Kirs
2023-10-11 10:04:19 +08:00
committed by GitHub
parent b7ac95a970
commit d4673ce28a
3 changed files with 34 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/scheduler/job/Job.java
View File

@ -151,6 +151,10 @@ public class Job implements Writable {
return jobStatus == JobStatus.STOPPED;
}
public boolean isFinished() {
return jobStatus == JobStatus.FINISHED;
}
public boolean isExpired(long nextExecuteTimestamp) {
if (endTimeMs == 0L) {
return false;

23
fe/fe-core/src/main/java/org/apache/doris/scheduler/manager/TimerJobManager.java
View File

@ -18,6 +18,7 @@
package org.apache.doris.scheduler.manager;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.PatternMatcher;
import org.apache.doris.common.io.Writable;
@ -273,9 +274,9 @@ public class TimerJobManager implements Closeable, Writable {
if (jobMap.get(jobId).getJobStatus().equals(JobStatus.FINISHED)) {
return;
}
job.setLatestCompleteExecuteTimeMs(System.currentTimeMillis());
cancelJobAllTask(job.getJobId());
job.setJobStatus(JobStatus.FINISHED);
jobMap.get(job.getJobId()).finish();
Env.getCurrentEnv().getEditLog().logUpdateJob(job);
}
@ -379,6 +380,7 @@ public class TimerJobManager implements Closeable, Writable {
log.info("re-register system scheduler tasks" + TimeUtils.longToTimeString(System.currentTimeMillis()));
dorisTimer.newTimeout(timeout -> {
batchSchedulerTasks();
clearFinishJob();
cycleSystemSchedulerTasks();
}, BATCH_SCHEDULER_INTERVAL_SECONDS, TimeUnit.SECONDS);
@ -510,4 +512,23 @@ public class TimerJobManager implements Closeable, Writable {
jobMap.putIfAbsent(job.getJobId(), job);
}
}
/**
* clear finish job，if job finish time is more than @Config.finish_job_max_saved_second, we will delete it
* this method will be called every 10 minutes, therefore, the actual maximum
* deletion time is Config.finish_job_max_saved_second + 10 min.
* we could to delete job in time, but it's not make sense.start
*/
private void clearFinishJob() {
Long now = System.currentTimeMillis();
jobMap.values().forEach(job -> {
if (job.isFinished() && now - job.getLatestCompleteExecuteTimeMs() > Config.finish_job_max_saved_second) {
jobMap.remove(job.getJobId());
Env.getCurrentEnv().getEditLog().logDeleteJob(job);
Env.getCurrentEnv().getJobTaskManager().deleteJobTasks(job.getJobId());
log.debug("delete finish job:{}", job.getJobId());
}
});
}
}