diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 2b8e341403..e62b226bf5 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1559,10 +1559,38 @@ public class Config extends ConfigBase { @ConfField public static boolean enable_pipeline_load = false; + /*---------------------- JOB CONFIG START------------------------*/ + /** + * The number of threads used to dispatch timer job. + * If we have a lot of timer jobs, we need more threads to dispatch them. + * All timer job will be dispatched to a thread pool, and they will be dispatched to the thread queue of the + * corresponding type of job + * The value should be greater than 0, if it is 0 or <=0, set it to 5 + */ + @ConfField(description = {"用于分发定时任务的线程数", + "The number of threads used to dispatch timer job."}) + public static int job_dispatch_timer_job_thread_num = 5; - @ConfField - public static int scheduler_job_task_max_saved_count = 20; + /** + * The number of timer jobs that can be queued. + * if consumer is slow, the queue will be full, and the producer will be blocked. + * if you have a lot of timer jobs, you need to increase this value or increase the number of + * {@code @dispatch_timer_job_thread_num} + * The value should be greater than 0, if it is 0 or <=0, set it to 1024 + */ + @ConfField(description = {"任务堆积时用于存放定时任务的队列大小", "The number of timer jobs that can be queued."}) + public static int job_dispatch_timer_job_queue_size = 1024; + /** + * The number of threads used to consume insert tasks. + * if you have a lot of insert jobs,and the average execution frequency is relatively high you need to increase + * this value or increase the number of {@code @job_insert_task_queue_size} + * The value should be greater than 0, if it is 0 or <=0, set it to 5 + */ + @ConfField(description = {"用于执行 Insert 任务的线程数", "The number of threads used to consume insert tasks."}) + public static int job_insert_task_consumer_thread_num = 10; + + /*---------------------- JOB CONFIG END------------------------*/ /** * The number of async tasks that can be queued. @See TaskDisruptor * if consumer is slow, the queue will be full, and the producer will be blocked. diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 9775b3c96d..ee65d926e8 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -2553,17 +2553,17 @@ resource_desc ::= create_job_stmt ::= KW_CREATE KW_JOB job_label:jobLabel KW_ON KW_SCHEDULER KW_EVERY INTEGER_LITERAL:time_interval ident:time_unit opt_job_starts:startsTime opt_job_ends:endsTime opt_comment:comment KW_DO stmt:executeSql {: - CreateJobStmt stmt = new CreateJobStmt(jobLabel,"RECURRING",null,time_interval,time_unit, startsTime, endsTime,comment,executeSql); + CreateJobStmt stmt = new CreateJobStmt(jobLabel,org.apache.doris.job.base.JobExecuteType.RECURRING,null,time_interval,time_unit, startsTime, endsTime,comment,executeSql); RESULT = stmt; :} - | KW_CREATE KW_JOB job_label:jobLabel KW_ON KW_SCHEDULER KW_STREAMING KW_AT STRING_LITERAL:atTime opt_comment:comment KW_DO stmt:executeSql +/* support in future | KW_CREATE KW_JOB job_label:jobLabel KW_ON KW_SCHEDULER KW_STREAMING KW_AT STRING_LITERAL:atTime opt_comment:comment KW_DO stmt:executeSql {: - CreateJobStmt stmt = new CreateJobStmt(jobLabel,"STREAMING",atTime,null,null,null,null,comment,executeSql); + CreateJobStmt stmt = new CreateJobStmt(jobLabel,org.apache.doris.job.base.JobExecuteType.STREAMING,atTime,null,null,null,null,comment,executeSql); RESULT = stmt; - :} + :} */ | KW_CREATE KW_JOB job_label:jobLabel KW_ON KW_SCHEDULER KW_AT STRING_LITERAL:atTime opt_comment:comment KW_DO stmt:executeSql {: - CreateJobStmt stmt = new CreateJobStmt(jobLabel,"ONE_TIME",atTime,null,null,null,null,comment,executeSql); + CreateJobStmt stmt = new CreateJobStmt(jobLabel,org.apache.doris.job.base.JobExecuteType.ONE_TIME,atTime,null,null,null,null,comment,executeSql); RESULT = stmt; :} ; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateJobStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateJobStmt.java index 57f976712c..d2aed10983 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateJobStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateJobStmt.java @@ -23,14 +23,15 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.job.base.AbstractJob; +import org.apache.doris.job.base.JobExecuteType; +import org.apache.doris.job.base.JobExecutionConfiguration; +import org.apache.doris.job.base.TimerDefinition; +import org.apache.doris.job.common.IntervalUnit; +import org.apache.doris.job.common.JobStatus; +import org.apache.doris.job.extensions.insert.InsertJob; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.scheduler.common.IntervalUnit; -import org.apache.doris.scheduler.constants.JobCategory; -import org.apache.doris.scheduler.constants.JobStatus; -import org.apache.doris.scheduler.constants.JobType; -import org.apache.doris.scheduler.executor.SqlJobExecutor; -import org.apache.doris.scheduler.job.Job; import com.google.common.collect.ImmutableSet; import lombok.Getter; @@ -61,12 +62,11 @@ import java.util.HashSet; @Slf4j public class CreateJobStmt extends DdlStmt { + @Getter + private StatementBase doStmt; @Getter - private StatementBase stmt; - - @Getter - private Job job; + private AbstractJob jobInstance; private final LabelName labelName; @@ -81,6 +81,7 @@ public class CreateJobStmt extends DdlStmt { private final String endsTimeStamp; private final String comment; + private JobExecuteType executeType; private String timezone = TimeUtils.DEFAULT_TIME_ZONE; @@ -88,9 +89,9 @@ public class CreateJobStmt extends DdlStmt { = new ImmutableSet.Builder>().add(InsertStmt.class) .add(UpdateStmt.class).build(); - private static HashSet supportStmtClassNamesCache = new HashSet<>(16); + private static final HashSet supportStmtClassNamesCache = new HashSet<>(16); - public CreateJobStmt(LabelName labelName, String jobTypeName, String onceJobStartTimestamp, + public CreateJobStmt(LabelName labelName, JobExecuteType executeType, String onceJobStartTimestamp, Long interval, String intervalTimeUnit, String startsTimeStamp, String endsTimeStamp, String comment, StatementBase doStmt) { this.labelName = labelName; @@ -100,10 +101,78 @@ public class CreateJobStmt extends DdlStmt { this.startsTimeStamp = startsTimeStamp; this.endsTimeStamp = endsTimeStamp; this.comment = comment; - this.stmt = doStmt; - this.job = new Job(); - JobType jobType = JobType.valueOf(jobTypeName.toUpperCase()); - job.setJobType(jobType); + this.doStmt = doStmt; + this.executeType = executeType; + } + + @Override + public void analyze(Analyzer analyzer) throws UserException { + super.analyze(analyzer); + checkAuth(); + labelName.analyze(analyzer); + String dbName = labelName.getDbName(); + Env.getCurrentInternalCatalog().getDbOrAnalysisException(dbName); + analyzerSqlStmt(); + // check its insert stmt,currently only support insert stmt + //todo used InsertIntoCommand if job is InsertJob + InsertJob job = new InsertJob(); + JobExecutionConfiguration jobExecutionConfiguration = new JobExecutionConfiguration(); + jobExecutionConfiguration.setExecuteType(executeType); + job.setCreateTimeMs(System.currentTimeMillis()); + TimerDefinition timerDefinition = new TimerDefinition(); + + if (null != onceJobStartTimestamp) { + timerDefinition.setStartTimeMs(TimeUtils.timeStringToLong(onceJobStartTimestamp)); + } + if (null != interval) { + timerDefinition.setInterval(interval); + } + if (null != intervalTimeUnit) { + timerDefinition.setIntervalUnit(IntervalUnit.valueOf(intervalTimeUnit.toUpperCase())); + } + if (null != startsTimeStamp) { + timerDefinition.setStartTimeMs(TimeUtils.timeStringToLong(startsTimeStamp)); + } + if (null != endsTimeStamp) { + timerDefinition.setEndTimeMs(TimeUtils.timeStringToLong(endsTimeStamp)); + } + jobExecutionConfiguration.setTimerDefinition(timerDefinition); + job.setJobConfig(jobExecutionConfiguration); + + job.setComment(comment); + job.setCurrentDbName(labelName.getDbName()); + job.setJobName(labelName.getLabelName()); + job.setCreateUser(ConnectContext.get().getCurrentUserIdentity()); + job.setJobStatus(JobStatus.RUNNING); + job.checkJobParams(); + String originStmt = getOrigStmt().originStmt; + String executeSql = parseExecuteSql(originStmt); + job.setExecuteSql(executeSql); + jobInstance = job; + } + + protected static void checkAuth() throws AnalysisException { + if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); + } + } + + private void checkStmtSupport() throws AnalysisException { + if (supportStmtClassNamesCache.contains(doStmt.getClass().getSimpleName())) { + return; + } + for (Class clazz : supportStmtSuperClass) { + if (clazz.isAssignableFrom(doStmt.getClass())) { + supportStmtClassNamesCache.add(doStmt.getClass().getSimpleName()); + return; + } + } + throw new AnalysisException("Not support this stmt type"); + } + + private void analyzerSqlStmt() throws UserException { + checkStmtSupport(); + doStmt.analyze(analyzer); } private String parseExecuteSql(String sql) throws AnalysisException { @@ -115,111 +184,4 @@ public class CreateJobStmt extends DdlStmt { } return executeSql; } - - @Override - public void analyze(Analyzer analyzer) throws UserException { - super.analyze(analyzer); - checkAuth(); - labelName.analyze(analyzer); - String dbName = labelName.getDbName(); - Env.getCurrentInternalCatalog().getDbOrAnalysisException(dbName); - job.setDbName(labelName.getDbName()); - job.setJobName(labelName.getLabelName()); - if (StringUtils.isNotBlank(onceJobStartTimestamp)) { - analyzerOnceTimeJob(); - } else { - analyzerCycleJob(); - } - if (ConnectContext.get() != null) { - timezone = ConnectContext.get().getSessionVariable().getTimeZone(); - } - timezone = TimeUtils.checkTimeZoneValidAndStandardize(timezone); - job.setTimezone(timezone); - job.setComment(comment); - //todo support user define - job.setUser(ConnectContext.get().getQualifiedUser()); - job.setJobStatus(JobStatus.RUNNING); - job.setJobCategory(JobCategory.SQL); - analyzerSqlStmt(); - } - - protected static void checkAuth() throws AnalysisException { - if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); - } - } - - private void checkStmtSupport() throws AnalysisException { - if (supportStmtClassNamesCache.contains(stmt.getClass().getSimpleName())) { - return; - } - for (Class clazz : supportStmtSuperClass) { - if (clazz.isAssignableFrom(stmt.getClass())) { - supportStmtClassNamesCache.add(stmt.getClass().getSimpleName()); - return; - } - } - throw new AnalysisException("Not support this stmt type"); - } - - private void analyzerSqlStmt() throws UserException { - checkStmtSupport(); - stmt.analyze(analyzer); - String originStmt = getOrigStmt().originStmt; - String executeSql = parseExecuteSql(originStmt); - SqlJobExecutor sqlJobExecutor = new SqlJobExecutor(executeSql); - job.setExecutor(sqlJobExecutor); - } - - - private void analyzerCycleJob() throws UserException { - if (null == interval) { - throw new AnalysisException("interval is null"); - } - if (interval <= 0) { - throw new AnalysisException("interval must be greater than 0"); - } - - if (StringUtils.isBlank(intervalTimeUnit)) { - throw new AnalysisException("intervalTimeUnit is null"); - } - try { - IntervalUnit intervalUnit = IntervalUnit.valueOf(intervalTimeUnit.toUpperCase()); - job.setIntervalUnit(intervalUnit); - long intervalTimeMs = intervalUnit.getParameterValue(interval); - job.setIntervalMs(intervalTimeMs); - job.setOriginInterval(interval); - } catch (IllegalArgumentException e) { - throw new AnalysisException("interval time unit is not valid, we only support second,minute,hour,day,week"); - } - if (StringUtils.isNotBlank(startsTimeStamp)) { - long startsTimeMillis = TimeUtils.timeStringToLong(startsTimeStamp); - if (startsTimeMillis < System.currentTimeMillis()) { - throw new AnalysisException("starts time must be greater than current time"); - } - job.setStartTimeMs(startsTimeMillis); - } - if (StringUtils.isNotBlank(endsTimeStamp)) { - long endTimeMillis = TimeUtils.timeStringToLong(endsTimeStamp); - if (endTimeMillis < System.currentTimeMillis()) { - throw new AnalysisException("ends time must be greater than current time"); - } - job.setEndTimeMs(endTimeMillis); - } - if (job.getStartTimeMs() > 0 && job.getEndTimeMs() > 0 - && (job.getEndTimeMs() - job.getStartTimeMs() < job.getIntervalMs())) { - throw new AnalysisException("ends time must be greater than start time and interval time"); - } - } - - - private void analyzerOnceTimeJob() throws UserException { - job.setIntervalMs(0L); - - long executeAtTimeMillis = TimeUtils.timeStringToLong(onceJobStartTimestamp); - if (executeAtTimeMillis < System.currentTimeMillis()) { - throw new AnalysisException("job time stamp must be greater than current time"); - } - job.setStartTimeMs(executeAtTimeMillis); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowJobStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowJobStmt.java index 42fb1c508f..dc4e96e05f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowJobStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowJobStmt.java @@ -26,12 +26,10 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSetMetaData; -import org.apache.doris.scheduler.constants.JobCategory; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import lombok.Getter; -import org.apache.commons.lang3.StringUtils; import java.util.List; @@ -52,7 +50,7 @@ public class ShowJobStmt extends ShowStmt { .add("ExecuteType") .add("RecurringStrategy") .add("Status") - .add("lastExecuteTaskStatus") + .add("ExecuteSql") .add("CreateTime") .add("Comment") .build(); @@ -65,9 +63,6 @@ public class ShowJobStmt extends ShowStmt { @Getter private String dbFullName; // optional - @Getter - private JobCategory jobCategory; // optional - private String jobCategoryName; // optional @Getter @@ -86,11 +81,6 @@ public class ShowJobStmt extends ShowStmt { super.analyze(analyzer); checkAuth(); checkLabelName(analyzer); - if (StringUtils.isBlank(jobCategoryName)) { - this.jobCategory = JobCategory.SQL; - } else { - this.jobCategory = JobCategory.valueOf(jobCategoryName.toUpperCase()); - } } private void checkAuth() throws AnalysisException { @@ -122,9 +112,6 @@ public class ShowJobStmt extends ShowStmt { ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); for (String title : TITLE_NAMES) { - if (this.jobCategory.equals(JobCategory.MTMV) && title.equals(NAME_TITLE)) { - builder.addColumn(new Column(MTMV_NAME_TITLE, ScalarType.createVarchar(30))); - } builder.addColumn(new Column(title, ScalarType.createVarchar(30))); } return builder.build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowJobTaskStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowJobTaskStmt.java index db3fb2ef3c..8d5c2b61db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowJobTaskStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowJobTaskStmt.java @@ -25,12 +25,10 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; import org.apache.doris.qe.ShowResultSetMetaData; -import org.apache.doris.scheduler.constants.JobCategory; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import lombok.Getter; -import org.apache.commons.lang3.StringUtils; import java.util.List; @@ -57,8 +55,6 @@ public class ShowJobTaskStmt extends ShowStmt { @Getter private final LabelName labelName; - @Getter - private JobCategory jobCategory; // optional @Getter private String dbFullName; // optional @@ -67,12 +63,6 @@ public class ShowJobTaskStmt extends ShowStmt { public ShowJobTaskStmt(String category, LabelName labelName) { this.labelName = labelName; - String jobCategoryName = category; - if (StringUtils.isBlank(jobCategoryName)) { - this.jobCategory = JobCategory.SQL; - } else { - this.jobCategory = JobCategory.valueOf(jobCategoryName.toUpperCase()); - } } @Override @@ -114,9 +104,6 @@ public class ShowJobTaskStmt extends ShowStmt { @Override public RedirectStatus getRedirectStatus() { - if (jobCategory.isPersistent()) { - return RedirectStatus.FORWARD_NO_SYNC; - } return RedirectStatus.NO_FORWARD; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 8c028bc8fc..1ed3291345 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -147,6 +147,8 @@ import org.apache.doris.ha.MasterInfo; import org.apache.doris.httpv2.entity.ResponseBody; import org.apache.doris.httpv2.meta.MetaBaseAction; import org.apache.doris.httpv2.rest.RestApiStatusCode; +import org.apache.doris.job.base.AbstractJob; +import org.apache.doris.job.manager.JobManager; import org.apache.doris.journal.JournalCursor; import org.apache.doris.journal.JournalEntity; import org.apache.doris.journal.bdbje.Timestamp; @@ -217,13 +219,8 @@ import org.apache.doris.qe.QueryCancelWorker; import org.apache.doris.qe.VariableMgr; import org.apache.doris.resource.Tag; import org.apache.doris.resource.workloadgroup.WorkloadGroupMgr; -import org.apache.doris.scheduler.disruptor.TaskDisruptor; -import org.apache.doris.scheduler.manager.JobTaskManager; -import org.apache.doris.scheduler.manager.TimerJobManager; import org.apache.doris.scheduler.manager.TransientTaskManager; import org.apache.doris.scheduler.registry.ExportTaskRegister; -import org.apache.doris.scheduler.registry.PersistentJobRegister; -import org.apache.doris.scheduler.registry.TimerJobRegister; import org.apache.doris.service.ExecuteEnv; import org.apache.doris.service.FrontendOptions; import org.apache.doris.statistics.AnalysisManager; @@ -349,13 +346,10 @@ public class Env { private CooldownConfHandler cooldownConfHandler; private MetastoreEventsProcessor metastoreEventsProcessor; - private PersistentJobRegister persistentJobRegister; private ExportTaskRegister exportTaskRegister; - private TimerJobManager timerJobManager; + private JobManager jobManager; private TransientTaskManager transientTaskManager; - private JobTaskManager jobTaskManager; - private TaskDisruptor taskDisruptor; private MasterDaemon labelCleaner; // To clean old LabelInfo, ExportJobInfos private MasterDaemon txnCleaner; // To clean aborted or timeout txns private Daemon feDiskUpdater; // Update fe disk info @@ -625,13 +619,8 @@ public class Env { this.cooldownConfHandler = new CooldownConfHandler(); } this.metastoreEventsProcessor = new MetastoreEventsProcessor(); - this.jobTaskManager = new JobTaskManager(); - this.timerJobManager = new TimerJobManager(); + this.jobManager = new JobManager<>(); this.transientTaskManager = new TransientTaskManager(); - this.taskDisruptor = new TaskDisruptor(this.timerJobManager, this.transientTaskManager); - this.timerJobManager.setDisruptor(taskDisruptor); - this.transientTaskManager.setDisruptor(taskDisruptor); - this.persistentJobRegister = new TimerJobRegister(timerJobManager); this.exportTaskRegister = new ExportTaskRegister(transientTaskManager); this.replayedJournalId = new AtomicLong(0L); this.stmtIdCounter = new AtomicLong(0L); @@ -1527,8 +1516,7 @@ public class Env { publishVersionDaemon.start(); // Start txn cleaner txnCleaner.start(); - taskDisruptor.start(); - timerJobManager.start(); + jobManager.start(); // Alter getAlterInstance().start(); // Consistency checker @@ -2008,29 +1996,17 @@ public class Env { } public long loadAsyncJobManager(DataInputStream in, long checksum) throws IOException { - timerJobManager.readFields(in); + jobManager.readFields(in); LOG.info("finished replay asyncJobMgr from image"); return checksum; } public long saveAsyncJobManager(CountingDataOutputStream out, long checksum) throws IOException { - timerJobManager.write(out); + jobManager.write(out); LOG.info("finished save analysisMgr to image"); return checksum; } - public long loadJobTaskManager(DataInputStream in, long checksum) throws IOException { - jobTaskManager.readFields(in); - LOG.info("finished replay jobTaskMgr from image"); - return checksum; - } - - public long saveJobTaskManager(CountingDataOutputStream out, long checksum) throws IOException { - jobTaskManager.write(out); - LOG.info("finished save jobTaskMgr to image"); - return checksum; - } - public long loadResources(DataInputStream in, long checksum) throws IOException { resourceMgr = ResourceMgr.read(in); LOG.info("finished replay resources from image"); @@ -3818,26 +3794,19 @@ public class Env { return this.syncJobManager; } - public PersistentJobRegister getJobRegister() { - return persistentJobRegister; - } public ExportTaskRegister getExportTaskRegister() { return exportTaskRegister; } - public TimerJobManager getAsyncJobManager() { - return timerJobManager; + public JobManager getJobManager() { + return jobManager; } public TransientTaskManager getTransientTaskManager() { return transientTaskManager; } - public JobTaskManager getJobTaskManager() { - return jobTaskManager; - } - public SmallFileMgr getSmallFileMgr() { return this.smallFileMgr; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java new file mode 100644 index 0000000000..6e2498cf01 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java @@ -0,0 +1,205 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.base; + +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; +import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.job.common.JobStatus; +import org.apache.doris.job.common.JobType; +import org.apache.doris.job.common.TaskStatus; +import org.apache.doris.job.exception.JobException; +import org.apache.doris.job.extensions.insert.InsertJob; +import org.apache.doris.job.task.AbstractTask; +import org.apache.doris.job.task.Task; + +import com.google.gson.annotations.SerializedName; +import lombok.Data; +import org.apache.commons.collections.CollectionUtils; + +import java.io.DataInput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@Data +public abstract class AbstractJob implements Job, Writable { + + @SerializedName(value = "jid") + private Long jobId; + + @SerializedName(value = "jn") + private String jobName; + + @SerializedName(value = "js") + private JobStatus jobStatus; + + @SerializedName(value = "cdb") + private String currentDbName; + + @SerializedName(value = "c") + private String comment; + + @SerializedName(value = "cu") + private UserIdentity createUser; + + @SerializedName(value = "jc") + private JobExecutionConfiguration jobConfig; + + @SerializedName(value = "ctms") + private Long createTimeMs; + + @SerializedName(value = "sql") + String executeSql; + + private List runningTasks = new ArrayList<>(); + + @Override + public void cancel() throws JobException { + if (CollectionUtils.isEmpty(runningTasks)) { + return; + } + runningTasks.forEach(Task::cancel); + + } + + @Override + public void cancel(long taskId) throws JobException { + if (CollectionUtils.isEmpty(runningTasks)) { + throw new JobException("no running task"); + } + runningTasks.stream().filter(task -> task.getTaskId().equals(taskId)).findFirst() + .orElseThrow(() -> new JobException("no task id:" + taskId)).cancel(); + } + + public void initTasks(List tasks) { + tasks.forEach(task -> { + task.setJobId(jobId); + task.setTaskId(Env.getCurrentEnv().getNextId()); + task.setCreateTimeMs(System.currentTimeMillis()); + task.setStatus(TaskStatus.PENDING); + }); + } + + public void checkJobParams() { + if (null == jobId) { + throw new IllegalArgumentException("jobId cannot be null"); + } + if (null == jobConfig) { + throw new IllegalArgumentException("jobConfig cannot be null"); + } + jobConfig.checkParams(createTimeMs); + checkJobParamsInternal(); + } + + public void updateJobStatus(JobStatus newJobStatus) { + if (null == newJobStatus) { + throw new IllegalArgumentException("jobStatus cannot be null"); + } + if (jobStatus == newJobStatus) { + throw new IllegalArgumentException(String.format("Can't update job %s status to the %s status", + jobStatus.name(), this.jobStatus.name())); + } + if (newJobStatus.equals(JobStatus.RUNNING) && !jobStatus.equals(JobStatus.PAUSED)) { + throw new IllegalArgumentException(String.format("Can't update job %s status to the %s status", + jobStatus.name(), this.jobStatus.name())); + } + if (newJobStatus.equals(JobStatus.STOPPED) && !jobStatus.equals(JobStatus.RUNNING)) { + throw new IllegalArgumentException(String.format("Can't update job %s status to the %s status", + jobStatus.name(), this.jobStatus.name())); + } + jobStatus = newJobStatus; + } + + + protected abstract void checkJobParamsInternal(); + + public static AbstractJob readFields(DataInput in) throws IOException { + // todo use RuntimeTypeAdapterFactory of Gson to do the serde + JobType jobType = JobType.valueOf(Text.readString(in)); + switch (jobType) { + case INSERT: + return InsertJob.readFields(in); + case MTMV: + // return MTMVJob.readFields(in); + break; + default: + throw new IllegalArgumentException("unknown job type"); + } + throw new IllegalArgumentException("unknown job type"); + } + + @Override + public void onTaskFail(T task) { + updateJobStatusIfEnd(); + } + + @Override + public void onTaskSuccess(T task) { + updateJobStatusIfEnd(); + runningTasks.remove(task); + + } + + private void updateJobStatusIfEnd() { + JobExecuteType executeType = getJobConfig().getExecuteType(); + if (executeType.equals(JobExecuteType.MANUAL)) { + return; + } + switch (executeType) { + case ONE_TIME: + case INSTANT: + jobStatus = JobStatus.FINISHED; + Env.getCurrentEnv().getJobManager().getJob(jobId).updateJobStatus(jobStatus); + break; + case RECURRING: + TimerDefinition timerDefinition = getJobConfig().getTimerDefinition(); + if (null != timerDefinition.getEndTimeMs() + && timerDefinition.getEndTimeMs() < System.currentTimeMillis() + + timerDefinition.getIntervalUnit().getIntervalMs(timerDefinition.getInterval())) { + jobStatus = JobStatus.FINISHED; + Env.getCurrentEnv().getJobManager().getJob(jobId).updateJobStatus(jobStatus); + } + break; + default: + break; + } + } + + /** + * get the job's common show info, which is used to show the job information + * eg:show jobs sql + * + * @return List job common show info + */ + public List getCommonShowInfo() { + List commonShowInfo = new ArrayList<>(); + commonShowInfo.add(String.valueOf(jobId)); + commonShowInfo.add(jobName); + commonShowInfo.add(createUser.getQualifiedUser()); + commonShowInfo.add(jobConfig.getExecuteType().name()); + commonShowInfo.add(jobConfig.convertRecurringStrategyToString()); + commonShowInfo.add(jobStatus.name()); + commonShowInfo.add(executeSql); + commonShowInfo.add(TimeUtils.longToTimeString(createTimeMs)); + commonShowInfo.add(comment); + return commonShowInfo; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/base/Job.java b/fe/fe-core/src/main/java/org/apache/doris/job/base/Job.java new file mode 100644 index 0000000000..25e93928d8 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/base/Job.java @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.base; + +import org.apache.doris.job.common.JobType; +import org.apache.doris.job.common.TaskType; +import org.apache.doris.job.exception.JobException; +import org.apache.doris.job.task.AbstractTask; +import org.apache.doris.qe.ShowResultSetMetaData; + +import java.util.List; + +/** + * The Job interface represents a job in the scheduler module, which stores the information of a job. + * A job can be uniquely identified using the job identifier. + * The job name is used for identification purposes and is not necessarily unique. + * The job status is used to control the execution of the job. + * + * @param The type of task associated with the job, extending AbstractTask. + */ +public interface Job { + + /** + * Creates a list of tasks of the specified type for this job. + * + * @param taskType The type of tasks to create. + * @return A list of tasks. + */ + List createTasks(TaskType taskType); + + /** + * Cancels the task with the specified taskId. + * + * @param taskId The ID of the task to cancel. + * @throws JobException If the task is not in the running state, it may have already + * finished and cannot be cancelled. + */ + void cancel(long taskId) throws JobException; + + /** + * Checks if the job is ready for scheduling. + * This method is called when starting the scheduled job, + * and if the job is not ready for scheduling, the scheduler will cancel it. + * + * @return True if the job is ready for scheduling, false otherwise. + */ + boolean isReadyForScheduling(); + + /** + * Retrieves the metadata for the job, which is used to display job information. + * + * @return The metadata for the job. + */ + ShowResultSetMetaData getJobMetaData(); + + /** + * Retrieves the metadata for the tasks, which is used to display task information. + * The metadata includes fields such as taskId, taskStatus, taskType, taskStartTime, taskEndTime, and taskProgress. + * + * @return The metadata for the tasks. + */ + ShowResultSetMetaData getTaskMetaData(); + + /** + * Retrieves the type of the job, which is used to identify different types of jobs. + * + * @return The type of the job. + */ + JobType getJobType(); + + /** + * Queries the list of tasks associated with this job. + * + * @return The list of tasks. + */ + List queryTasks(); + + /** + * Cancels all running tasks of this job. + * + * @throws JobException If cancelling a running task fails. + */ + void cancel() throws JobException; + + /** + * Notifies the job when a task execution fails. + * + * @param task The failed task. + */ + void onTaskFail(T task); + + /** + * Notifies the job when a task execution is successful. + * + * @param task The successful task. + */ + void onTaskSuccess(T task); + + /** + * Notifies the job when a task execution is cancelled. + * + * @param task The cancelled task. + */ + void onTaskCancel(T task); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/constants/JobType.java b/fe/fe-core/src/main/java/org/apache/doris/job/base/JobExecuteType.java similarity index 88% rename from fe/fe-core/src/main/java/org/apache/doris/scheduler/constants/JobType.java rename to fe/fe-core/src/main/java/org/apache/doris/job/base/JobExecuteType.java index 58f681c406..ea9ddb3b02 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/constants/JobType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/base/JobExecuteType.java @@ -15,9 +15,10 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.scheduler.constants; +package org.apache.doris.job.base; + +public enum JobExecuteType { -public enum JobType { /** * The job will be executed only once. */ @@ -34,5 +35,9 @@ public enum JobType { /** * The job will be executed manually and need to be triggered by the user. */ - MANUAL + MANUAL, + /** + * The job will be executed immediately. + */ + INSTANT, } diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/base/JobExecutionConfiguration.java b/fe/fe-core/src/main/java/org/apache/doris/job/base/JobExecutionConfiguration.java new file mode 100644 index 0000000000..ead943d4bf --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/base/JobExecutionConfiguration.java @@ -0,0 +1,211 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.base; + +import org.apache.doris.common.util.TimeUtils; + +import com.google.gson.annotations.SerializedName; +import lombok.Getter; +import lombok.Setter; + +import java.util.ArrayList; +import java.util.List; + + +public class JobExecutionConfiguration { + + @Getter + @Setter + @SerializedName(value = "td") + private TimerDefinition timerDefinition; + @Getter + @Setter + @SerializedName(value = "ec") + private JobExecuteType executeType; + + /** + * Maximum number of concurrent tasks, <= 0 means no limit + * if the number of tasks exceeds the limit, the task will be delayed execution + * todo: implement this later, we need to consider concurrency strategies + */ + @SerializedName(value = "maxConcurrentTaskNum") + private Integer maxConcurrentTaskNum; + + public void checkParams(Long createTimeMs) { + if (executeType == null) { + throw new IllegalArgumentException("executeType cannot be null"); + } + + if (executeType == JobExecuteType.INSTANT || executeType == JobExecuteType.MANUAL) { + return; + } + + checkTimerDefinition(createTimeMs); + + if (executeType == JobExecuteType.ONE_TIME) { + validateStartTimeMs(); + return; + } + + if (executeType == JobExecuteType.STREAMING) { + validateStartTimeMs(); + return; + } + + if (executeType == JobExecuteType.RECURRING) { + if (timerDefinition.getInterval() == null) { + throw new IllegalArgumentException("interval cannot be null when executeType is RECURRING"); + } + if (timerDefinition.getIntervalUnit() == null) { + throw new IllegalArgumentException("intervalUnit cannot be null when executeType is RECURRING"); + } + } + } + + private void checkTimerDefinition(long createTimeMs) { + if (timerDefinition == null) { + throw new IllegalArgumentException( + "timerDefinition cannot be null when executeType is not instant or manual"); + } + timerDefinition.checkParams(createTimeMs); + } + + private void validateStartTimeMs() { + if (timerDefinition.getStartTimeMs() == null) { + throw new IllegalArgumentException("startTimeMs cannot be null"); + } + if (timerDefinition.getStartTimeMs() < System.currentTimeMillis()) { + throw new IllegalArgumentException("startTimeMs cannot be less than current time"); + } + } + + + // Returns a list of delay times in seconds for triggering the job + public List getTriggerDelayTimes(Long currentTimeMs, Long startTimeMs, Long endTimeMs) { + List delayTimeSeconds = new ArrayList<>(); + + if (JobExecuteType.ONE_TIME.equals(executeType)) { + // If the job is already executed or in the schedule queue, or not within this schedule window + if (null != timerDefinition.getLatestSchedulerTimeMs() || endTimeMs < timerDefinition.getStartTimeMs()) { + return delayTimeSeconds; + } + + delayTimeSeconds.add(queryDelayTimeSecond(currentTimeMs, timerDefinition.getStartTimeMs())); + this.timerDefinition.setLatestSchedulerTimeMs(timerDefinition.getStartTimeMs()); + return delayTimeSeconds; + } + + if (JobExecuteType.STREAMING.equals(executeType) && null != timerDefinition) { + if (null == timerDefinition.getStartTimeMs() || null != timerDefinition.getLatestSchedulerTimeMs()) { + return delayTimeSeconds; + } + + // If the job is already executed or in the schedule queue, or not within this schedule window + if (endTimeMs < timerDefinition.getStartTimeMs()) { + return delayTimeSeconds; + } + + delayTimeSeconds.add(queryDelayTimeSecond(currentTimeMs, timerDefinition.getStartTimeMs())); + this.timerDefinition.setLatestSchedulerTimeMs(timerDefinition.getStartTimeMs()); + return delayTimeSeconds; + } + + if (JobExecuteType.RECURRING.equals(executeType)) { + if (timerDefinition.getStartTimeMs() > endTimeMs || null != timerDefinition.getEndTimeMs() + && timerDefinition.getEndTimeMs() < startTimeMs) { + return delayTimeSeconds; + } + + return getExecutionDelaySeconds(startTimeMs, endTimeMs, timerDefinition.getStartTimeMs(), + timerDefinition.getIntervalUnit().getIntervalMs(timerDefinition.getInterval()), currentTimeMs); + } + + return delayTimeSeconds; + } + + // Returns the delay time in seconds between the current time and the specified start time + private Long queryDelayTimeSecond(Long currentTimeMs, Long startTimeMs) { + if (startTimeMs <= currentTimeMs) { + return 0L; + } + + return (startTimeMs - currentTimeMs) / 1000; + } + + // Returns a list of delay times in seconds for executing the job within the specified window + private List getExecutionDelaySeconds(long windowStartTimeMs, long windowEndTimeMs, long startTimeMs, + long intervalMs, long currentTimeMs) { + List timestamps = new ArrayList<>(); + + long windowDuration = windowEndTimeMs - windowStartTimeMs; + + if (windowDuration <= 0 || intervalMs <= 0) { + return timestamps; // Return an empty list if there won't be any trigger time + } + + long firstTriggerTime = windowStartTimeMs + (intervalMs - ((windowStartTimeMs - startTimeMs) + % intervalMs)) % intervalMs; + + if (firstTriggerTime < currentTimeMs) { + firstTriggerTime += intervalMs; + } + + if (firstTriggerTime > windowEndTimeMs) { + return timestamps; // Return an empty list if there won't be any trigger time + } + + // Calculate the trigger time list + for (long triggerTime = firstTriggerTime; triggerTime <= windowEndTimeMs; triggerTime += intervalMs) { + if (triggerTime >= currentTimeMs && (null == timerDefinition.getEndTimeMs() + || triggerTime < timerDefinition.getEndTimeMs())) { + timestamps.add(queryDelayTimeSecond(currentTimeMs, triggerTime)); + } + } + + return timestamps; + } + + public String convertRecurringStrategyToString() { + switch (executeType) { + case ONE_TIME: + return "AT " + TimeUtils.longToTimeString(timerDefinition.getStartTimeMs()); + case RECURRING: + String result = "EVERY " + timerDefinition.getInterval() + " " + + timerDefinition.getIntervalUnit().name() + " STARTS " + + TimeUtils.longToTimeString(timerDefinition.getStartTimeMs()); + + if (null != timerDefinition.getEndTimeMs()) { + result += " ENDS " + TimeUtils.longToTimeString(timerDefinition.getEndTimeMs()); + } + return result; + /* case STREAMING: + return "STREAMING" + (startTimeMs > 0 ? " AT " + TimeUtils.longToTimeString(startTimeMs) : "");*/ + case MANUAL: + return "MANUAL TRIGGER"; + case INSTANT: + return "INSTANT"; + default: + return "UNKNOWN"; + } + } + + public boolean checkIsTimerJob() { + return null != timerDefinition; + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/base/TimerDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/job/base/TimerDefinition.java new file mode 100644 index 0000000000..153bc2c43c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/base/TimerDefinition.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.base; + +import org.apache.doris.job.common.IntervalUnit; + +import com.google.gson.annotations.SerializedName; +import lombok.Data; + +@Data +public class TimerDefinition { + + @SerializedName(value = "il") + private Long interval; + + @SerializedName(value = "iu") + private IntervalUnit intervalUnit; + @SerializedName(value = "stm") + private Long startTimeMs; + @SerializedName(value = "etm") + private Long endTimeMs; + + private Long latestSchedulerTimeMs; + + + public void checkParams(Long createTimeMs) { + if (null != startTimeMs && startTimeMs < System.currentTimeMillis()) { + throw new IllegalArgumentException("startTimeMs must be greater than current time"); + } + if (null == startTimeMs) { + startTimeMs = createTimeMs + intervalUnit.getIntervalMs(interval); + } + if (null != endTimeMs && endTimeMs < startTimeMs) { + throw new IllegalArgumentException("end time cannot be less than start time"); + } + + if (null != intervalUnit) { + if (null == interval) { + throw new IllegalArgumentException("interval cannot be null when intervalUnit is not null"); + } + if (interval <= 0) { + throw new IllegalArgumentException("interval must be greater than 0"); + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/common/IntervalUnit.java b/fe/fe-core/src/main/java/org/apache/doris/job/common/IntervalUnit.java similarity index 92% rename from fe/fe-core/src/main/java/org/apache/doris/scheduler/common/IntervalUnit.java rename to fe/fe-core/src/main/java/org/apache/doris/job/common/IntervalUnit.java index 27f10d8a3f..4c576e986f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/common/IntervalUnit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/common/IntervalUnit.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.scheduler.common; +package org.apache.doris.job.common; import java.util.Arrays; import java.util.concurrent.TimeUnit; @@ -60,7 +60,7 @@ public enum IntervalUnit { .orElseThrow(() -> new IllegalArgumentException("Unknown configuration " + name)); } - public Long getParameterValue(Long param) { - return (Long) (param != null ? converter.apply(param) : defaultValue); + public Long getIntervalMs(Long interval) { + return (Long) (interval != null ? converter.apply(interval) : defaultValue); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/constants/JobStatus.java b/fe/fe-core/src/main/java/org/apache/doris/job/common/JobStatus.java similarity index 96% rename from fe/fe-core/src/main/java/org/apache/doris/scheduler/constants/JobStatus.java rename to fe/fe-core/src/main/java/org/apache/doris/job/common/JobStatus.java index f01686a521..2df65e4654 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/constants/JobStatus.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/common/JobStatus.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.scheduler.constants; +package org.apache.doris.job.common; public enum JobStatus { @@ -42,3 +42,4 @@ public enum JobStatus { */ FINISHED } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/common/JobType.java b/fe/fe-core/src/main/java/org/apache/doris/job/common/JobType.java new file mode 100644 index 0000000000..f815e16cec --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/common/JobType.java @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.common; + +public enum JobType { + INSERT, + MTMV +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/common/TaskStatus.java b/fe/fe-core/src/main/java/org/apache/doris/job/common/TaskStatus.java new file mode 100644 index 0000000000..b4040d31e0 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/common/TaskStatus.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.common; + +public enum TaskStatus { + PENDING, + CANCEL, + RUNNING, + SUCCESS, + FAILD; +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/common/TaskType.java b/fe/fe-core/src/main/java/org/apache/doris/job/common/TaskType.java new file mode 100644 index 0000000000..7f782cb412 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/common/TaskType.java @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.common; + +public enum TaskType { + + SCHEDULED, + MANUAL; +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/ExecuteTaskEvent.java b/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/ExecuteTaskEvent.java new file mode 100644 index 0000000000..3d8f9ed153 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/ExecuteTaskEvent.java @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.disruptor; + +import org.apache.doris.job.base.JobExecutionConfiguration; +import org.apache.doris.job.task.AbstractTask; + +import com.lmax.disruptor.EventFactory; +import lombok.Data; + +@Data +public class ExecuteTaskEvent { + + private T task; + + private JobExecutionConfiguration jobConfig; + + public static EventFactory> factory() { + return ExecuteTaskEvent::new; + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/TaskDisruptor.java b/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/TaskDisruptor.java new file mode 100644 index 0000000000..0994a1d921 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/TaskDisruptor.java @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.disruptor; + +import com.lmax.disruptor.EventFactory; +import com.lmax.disruptor.EventTranslatorVararg; +import com.lmax.disruptor.RingBuffer; +import com.lmax.disruptor.WaitStrategy; +import com.lmax.disruptor.WorkHandler; +import com.lmax.disruptor.dsl.Disruptor; +import com.lmax.disruptor.dsl.ProducerType; + +import java.util.concurrent.ThreadFactory; + +/** + * Utility class for creating and managing a Disruptor instance. + * + * @param the type of the event handled by the Disruptor + */ +public class TaskDisruptor { + private final Disruptor disruptor; + private final EventTranslatorVararg eventTranslator; + + /** + * Constructs a DisruptorUtil instance. + * + * @param eventFactory the factory for creating events + * @param ringBufferSize the size of the ring buffer + * @param threadFactory the thread factory to create threads for event handling + * @param waitStrategy the wait strategy for the ring buffer + * @param workHandlers the work handlers for processing events + * @param eventTranslator the translator for publishing events with variable arguments + */ + public TaskDisruptor(EventFactory eventFactory, int ringBufferSize, ThreadFactory threadFactory, + WaitStrategy waitStrategy, WorkHandler[] workHandlers, + EventTranslatorVararg eventTranslator) { + disruptor = new Disruptor<>(eventFactory, ringBufferSize, threadFactory, + ProducerType.SINGLE, waitStrategy); + disruptor.handleEventsWithWorkerPool(workHandlers); + this.eventTranslator = eventTranslator; + disruptor.start(); + } + + /** + * Starts the Disruptor. + */ + public void start() { + disruptor.start(); + } + + /** + * Publishes an event with the provided arguments. + * + * @param args the arguments for the event + */ + public void publishEvent(Object... args) { + RingBuffer ringBuffer = disruptor.getRingBuffer(); + ringBuffer.publishEvent(eventTranslator, args); + } + + /** + * Shuts down the Disruptor. + */ + public void shutdown() { + disruptor.shutdown(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/TimerJobEvent.java b/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/TimerJobEvent.java new file mode 100644 index 0000000000..218fefd041 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/TimerJobEvent.java @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.disruptor; + +import org.apache.doris.job.base.AbstractJob; + +import com.lmax.disruptor.EventFactory; +import lombok.Data; + +@Data +public class TimerJobEvent> { + + + private T job; + + public static > EventFactory> factory() { + return TimerJobEvent::new; + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/exception/JobException.java b/fe/fe-core/src/main/java/org/apache/doris/job/exception/JobException.java new file mode 100644 index 0000000000..d6e93ead62 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/exception/JobException.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.exception; + +/** + * This class represents a job exception that can be thrown when a job is executed. + */ +public class JobException extends Exception { + public JobException(String message) { + super(message); + } + + public JobException(String format, Object... msg) { + super(String.format(format, msg)); + } + + public JobException(String message, Throwable cause) { + super(message, cause); + } + + public JobException(Throwable cause) { + super(cause); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/executor/DefaultTaskExecutorHandler.java b/fe/fe-core/src/main/java/org/apache/doris/job/executor/DefaultTaskExecutorHandler.java new file mode 100644 index 0000000000..0e5911a4e8 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/executor/DefaultTaskExecutorHandler.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.executor; + +import org.apache.doris.job.disruptor.ExecuteTaskEvent; +import org.apache.doris.job.task.AbstractTask; + +import com.lmax.disruptor.WorkHandler; +import lombok.extern.slf4j.Slf4j; + +/** + * DefaultTaskExecutor is an implementation of the TaskExecutor interface. + * if you need to implement your own TaskExecutor, you could refer to this class. and need to register + * it in the TaskExecutorFactory + * It executes a given AbstractTask by acquiring a semaphore token from the TaskTokenManager + * and releasing it after the task execution. + */ +@Slf4j +public class DefaultTaskExecutorHandler implements WorkHandler> { + + + @Override + public void onEvent(ExecuteTaskEvent executeTaskEvent) { + T task = executeTaskEvent.getTask(); + if (null == task) { + log.warn("task is null, ignore,maybe task has been canceled"); + return; + } + if (task.isCancelled()) { + log.info("task is canceled, ignore"); + return; + } + try { + task.runTask(); + } catch (Exception e) { + //if task.onFail() throw exception, we will catch it here + log.warn("task before error, task id is {}", task.getTaskId(), e); + } + //todo we need discuss whether we need to use semaphore to control the concurrent task num + /* Semaphore semaphore = null; + // get token + try { + int maxConcurrentTaskNum = executeTaskEvent.getJobConfig().getMaxConcurrentTaskNum(); + semaphore = TaskTokenManager.tryAcquire(task.getJobId(), maxConcurrentTaskNum); + task.runTask(); + } catch (Exception e) { + task.onFail(); + log.error("execute task error, task id is {}", task.getTaskId(), e); + } finally { + if (null != semaphore) { + semaphore.release(); + }*/ + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/executor/DispatchTaskHandler.java b/fe/fe-core/src/main/java/org/apache/doris/job/executor/DispatchTaskHandler.java new file mode 100644 index 0000000000..852ba13415 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/executor/DispatchTaskHandler.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.executor; + +import org.apache.doris.job.base.AbstractJob; +import org.apache.doris.job.common.JobStatus; +import org.apache.doris.job.common.JobType; +import org.apache.doris.job.common.TaskType; +import org.apache.doris.job.disruptor.TaskDisruptor; +import org.apache.doris.job.disruptor.TimerJobEvent; +import org.apache.doris.job.task.AbstractTask; + +import com.lmax.disruptor.WorkHandler; +import jline.internal.Log; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; +import java.util.Map; + +/** + * dispatch timer job to task disruptor + * when job is ready for scheduling and job status is running + * we will create task and publish to task disruptor @see DefaultTaskExecutorHandler + */ +@Slf4j +public class DispatchTaskHandler> implements WorkHandler> { + + private final Map> disruptorMap; + + public DispatchTaskHandler(Map> disruptorMap) { + this.disruptorMap = disruptorMap; + } + + + @Override + public void onEvent(TimerJobEvent event) { + try { + if (null == event.getJob()) { + log.info("job is null,may be job is deleted, ignore"); + return; + } + if (event.getJob().isReadyForScheduling() && event.getJob().getJobStatus() == JobStatus.RUNNING) { + List tasks = event.getJob().createTasks(TaskType.SCHEDULED); + JobType jobType = event.getJob().getJobType(); + for (AbstractTask task : tasks) { + disruptorMap.get(jobType).publishEvent(task, event.getJob().getJobConfig()); + } + } + } catch (Exception e) { + Log.warn("dispatch timer job error, task id is {}", event.getJob().getJobId(), e); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/executor/TimerJobSchedulerTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/executor/TimerJobSchedulerTask.java new file mode 100644 index 0000000000..9bd4cbfeae --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/executor/TimerJobSchedulerTask.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.executor; + +import org.apache.doris.job.base.AbstractJob; +import org.apache.doris.job.disruptor.TaskDisruptor; + +import io.netty.util.Timeout; +import io.netty.util.TimerTask; +import jline.internal.Log; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class TimerJobSchedulerTask> implements TimerTask { + + private TaskDisruptor dispatchDisruptor; + + private final T job; + + public TimerJobSchedulerTask(TaskDisruptor dispatchDisruptor, T job) { + this.dispatchDisruptor = dispatchDisruptor; + this.job = job; + } + + @Override + public void run(Timeout timeout) { + try { + dispatchDisruptor.publishEvent(this.job); + } catch (Exception e) { + Log.warn("dispatch timer job error, task id is {}", this.job.getJobId(), e); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/job/ExecutorResult.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertIntoState.java similarity index 77% rename from fe/fe-core/src/main/java/org/apache/doris/scheduler/job/ExecutorResult.java rename to fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertIntoState.java index 99df9c9e78..3b78280857 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/job/ExecutorResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertIntoState.java @@ -15,21 +15,11 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.scheduler.job; +package org.apache.doris.job.extensions.insert; -import lombok.AllArgsConstructor; import lombok.Data; @Data -@AllArgsConstructor -public class ExecutorResult { - - private T result; - - private boolean success; - - private String errorMsg; - - private String executorSql; - +public class InsertIntoState { + private String trackingUrl; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertJob.java new file mode 100644 index 0000000000..70a558009d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertJob.java @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.extensions.insert; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.io.Text; +import org.apache.doris.job.base.AbstractJob; +import org.apache.doris.job.common.JobType; +import org.apache.doris.job.common.TaskType; +import org.apache.doris.job.exception.JobException; +import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.qe.ShowResultSetMetaData; + +import com.google.gson.annotations.SerializedName; +import lombok.Data; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +@Data +public class InsertJob extends AbstractJob { + + @SerializedName(value = "labelPrefix") + String labelPrefix; + + + @Override + public List createTasks(TaskType taskType) { + InsertTask task = new InsertTask(null, null, null, null, null); + task.setJobId(getJobId()); + task.setTaskType(taskType); + task.setTaskId(Env.getCurrentEnv().getNextId()); + ArrayList tasks = new ArrayList<>(); + tasks.add(task); + super.initTasks(tasks); + getRunningTasks().addAll(tasks); + return tasks; + } + + @Override + public void cancel(long taskId) throws JobException { + super.cancel(); + } + + + @Override + public void cancel() throws JobException { + super.cancel(); + } + + @Override + public boolean isReadyForScheduling() { + return true; + } + + + @Override + protected void checkJobParamsInternal() { + + } + + public static InsertJob readFields(DataInput in) throws IOException { + return GsonUtils.GSON.fromJson(Text.readString(in), InsertJob.class); + } + + @Override + public List queryTasks() { + return null; + } + + @Override + public JobType getJobType() { + return JobType.INSERT; + } + + @Override + public ShowResultSetMetaData getJobMetaData() { + return null; + } + + @Override + public ShowResultSetMetaData getTaskMetaData() { + return null; + } + + @Override + public void onTaskFail(InsertTask task) { + getRunningTasks().remove(task); + } + + @Override + public void onTaskSuccess(InsertTask task) { + getRunningTasks().remove(task); + } + + @Override + public void onTaskCancel(InsertTask task) { + getRunningTasks().remove(task); + } + + + @Override + public void write(DataOutput out) throws IOException { + Text.writeString(out, JobType.INSERT.name()); + Text.writeString(out, GsonUtils.GSON.toJson(this)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java new file mode 100644 index 0000000000..9802ebc55d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.extensions.insert; + +import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.job.task.AbstractTask; +import org.apache.doris.load.FailMsg; +import org.apache.doris.load.loadv2.LoadJob; +import org.apache.doris.nereids.trees.plans.commands.InsertIntoTableCommand; + +import lombok.extern.slf4j.Slf4j; + +/** + * todo implement this later + */ +@Slf4j +public class InsertTask extends AbstractTask { + + private String labelName; + + private InsertIntoTableCommand command; + private LoadJob.LoadStatistic statistic; + private FailMsg failMsg; + + private InsertIntoState insertIntoState; + + @Override + public void before() { + super.before(); + } + + public InsertTask(String labelName, InsertIntoTableCommand command, LoadJob.LoadStatistic statistic, + FailMsg failMsg, InsertIntoState insertIntoState) { + this.labelName = labelName; + this.command = command; + this.statistic = statistic; + this.failMsg = failMsg; + this.insertIntoState = insertIntoState; + } + + @Override + public void run() { + //just for test + log.info(getJobId() + "InsertTask run" + TimeUtils.longToTimeString(System.currentTimeMillis())); + } + + @Override + public void onFail() { + super.onFail(); + } + + @Override + public void onSuccess() { + super.onSuccess(); + } + + @Override + public void cancel() { + super.cancel(); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java b/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java new file mode 100644 index 0000000000..0df87a8451 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java @@ -0,0 +1,216 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.manager; + +import org.apache.doris.common.io.Writable; +import org.apache.doris.common.util.LogBuilder; +import org.apache.doris.common.util.LogKey; +import org.apache.doris.job.base.AbstractJob; +import org.apache.doris.job.common.JobStatus; +import org.apache.doris.job.common.JobType; +import org.apache.doris.job.common.TaskType; +import org.apache.doris.job.exception.JobException; +import org.apache.doris.job.scheduler.JobScheduler; +import org.apache.doris.job.task.AbstractTask; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +@Slf4j +public class JobManager> implements Writable { + + + private final ConcurrentHashMap jobMap = new ConcurrentHashMap<>(32); + + private JobScheduler jobScheduler; + + public void start() { + jobScheduler = new JobScheduler(jobMap); + jobScheduler.start(); + } + + public void registerJob(T job) throws JobException { + job.checkJobParams(); + checkJobNameExist(job.getJobName(), job.getJobType(), job.getCurrentDbName()); + if (jobMap.get(job.getJobId()) != null) { + throw new JobException("job id exist,jobId:" + job.getJobId()); + } + //Env.getCurrentEnv().getEditLog().logCreateJob(job); + //check name exist + jobMap.put(job.getJobId(), job); + //check its need to scheduler + jobScheduler.scheduleOneJob(job); + } + + + private void checkJobNameExist(String jobName, JobType type, String currentDbName) throws JobException { + if (jobMap.values().stream().anyMatch(a -> a.getJobName().equals(jobName) && a.getJobType().equals(type) + && (null == a.getCurrentDbName() || a.getCurrentDbName().equals(currentDbName)))) { + throw new JobException("job name exist,jobName:" + jobName); + } + } + + public void unregisterJob(Long jobId) throws JobException { + checkJobExist(jobId); + jobMap.get(jobId).setJobStatus(JobStatus.STOPPED); + jobMap.get(jobId).cancel(); + //Env.getCurrentEnv().getEditLog().logDeleteJob(jobMap.get(jobId)); + jobMap.remove(jobId); + } + + public void unregisterJob(String currentDbName, String jobName) throws JobException { + for (T a : jobMap.values()) { + if (a.getJobName().equals(jobName) && (null != a.getCurrentDbName() + && a.getCurrentDbName().equals(currentDbName)) && a.getJobType().equals(JobType.INSERT)) { + try { + unregisterJob(a.getJobId()); + } catch (JobException e) { + throw new JobException("unregister job error,jobName:" + jobName); + } + } + } + + } + + public void alterJobStatus(Long jobId, JobStatus status) throws JobException { + checkJobExist(jobId); + jobMap.get(jobId).updateJobStatus(status); + //Env.getCurrentEnv().getEditLog().logUpdateJob(jobMap.get(jobId)); + } + + public void alterJobStatus(String currentDbName, String jobName, JobStatus jobStatus) throws JobException { + for (T a : jobMap.values()) { + if (a.getJobName().equals(jobName) && (null != a.getCurrentDbName() + && a.getCurrentDbName().equals(currentDbName)) && JobType.INSERT.equals(a.getJobType())) { + try { + alterJobStatus(a.getJobId(), jobStatus); + } catch (JobException e) { + throw new JobException("unregister job error,jobName:" + jobName); + } + } + } + } + + private void checkJobExist(Long jobId) throws JobException { + if (null == jobMap.get(jobId)) { + throw new JobException("job not exist,jobId:" + jobId); + } + } + + public List queryJobs(JobType type) { + return jobMap.values().stream().filter(a -> a.getJobType().equals(type)) + .collect(java.util.stream.Collectors.toList()); + } + + public List queryJobs(String currentDb, String jobName) { + //only query insert job,we just provide insert job + return jobMap.values().stream().filter(a -> checkItsMatch(currentDb, jobName, a)) + .collect(Collectors.toList()); + } + + private boolean checkItsMatch(String currentDb, String jobName, T job) { + if (StringUtils.isBlank(jobName)) { + return job.getJobType().equals(JobType.INSERT) && (null != job.getCurrentDbName() + && job.getCurrentDbName().equals(currentDb)); + } + return job.getJobType().equals(JobType.INSERT) && (null != job.getCurrentDbName() + && job.getCurrentDbName().equals(currentDb)) && job.getJobName().equals(jobName); + } + + public List queryTasks(Long jobId) throws JobException { + checkJobExist(jobId); + return jobMap.get(jobId).queryTasks(); + } + + public void triggerJob(long jobId) throws JobException { + checkJobExist(jobId); + jobScheduler.schedulerInstantJob(jobMap.get(jobId), TaskType.MANUAL); + } + + public void replayCreateJob(T job) { + if (jobMap.containsKey(job.getJobId())) { + return; + } + jobMap.putIfAbsent(job.getJobId(), job); + log.info(new LogBuilder(LogKey.SCHEDULER_JOB, job.getJobId()) + .add("msg", "replay create scheduler job").build()); + } + + /** + * Replay update load job. + **/ + public void replayUpdateJob(T job) { + jobMap.put(job.getJobId(), job); + log.info(new LogBuilder(LogKey.SCHEDULER_JOB, job.getJobId()) + .add("msg", "replay update scheduler job").build()); + } + + public void replayDeleteJob(T job) { + if (null == jobMap.get(job.getJobId())) { + return; + } + jobMap.remove(job.getJobId()); + log.info(new LogBuilder(LogKey.SCHEDULER_JOB, job.getJobId()) + .add("msg", "replay delete scheduler job").build()); + } + + void cancelTask(Long jobId, Long taskId) throws JobException { + checkJobExist(jobId); + if (null == jobMap.get(jobId).getRunningTasks()) { + throw new JobException("task not exist,taskId:" + taskId); + } + } + + @Override + public void write(DataOutput out) throws IOException { + out.writeInt(jobMap.size()); + jobMap.forEach((jobId, job) -> { + try { + job.write(out); + } catch (IOException e) { + log.error("write job error,jobId:" + jobId, e); + } + }); + } + + /** + * read job from data input, and init job + * + * @param in data input + * @throws IOException io exception when read data input error + */ + public void readFields(DataInput in) throws IOException { + int size = in.readInt(); + for (int i = 0; i < size; i++) { + AbstractJob job = AbstractJob.readFields(in); + jobMap.putIfAbsent(job.getJobId(), (T) job); + } + } + + public T getJob(Long jobId) { + return jobMap.get(jobId); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/manager/TaskDisruptorGroupManager.java b/fe/fe-core/src/main/java/org/apache/doris/job/manager/TaskDisruptorGroupManager.java new file mode 100644 index 0000000000..1becb12d70 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/manager/TaskDisruptorGroupManager.java @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.manager; + +import org.apache.doris.common.Config; +import org.apache.doris.common.CustomThreadFactory; +import org.apache.doris.job.base.AbstractJob; +import org.apache.doris.job.base.JobExecutionConfiguration; +import org.apache.doris.job.common.JobType; +import org.apache.doris.job.disruptor.ExecuteTaskEvent; +import org.apache.doris.job.disruptor.TaskDisruptor; +import org.apache.doris.job.disruptor.TimerJobEvent; +import org.apache.doris.job.executor.DefaultTaskExecutorHandler; +import org.apache.doris.job.executor.DispatchTaskHandler; +import org.apache.doris.job.extensions.insert.InsertTask; +import org.apache.doris.job.task.AbstractTask; + +import com.lmax.disruptor.BlockingWaitStrategy; +import com.lmax.disruptor.EventFactory; +import com.lmax.disruptor.EventTranslatorVararg; +import com.lmax.disruptor.WorkHandler; +import lombok.Getter; + +import java.util.EnumMap; +import java.util.Map; +import java.util.concurrent.ThreadFactory; + +public class TaskDisruptorGroupManager { + + private final Map> disruptorMap = new EnumMap<>(JobType.class); + + @Getter + private TaskDisruptor>> dispatchDisruptor; + + private static final int DEFAULT_RING_BUFFER_SIZE = 1024; + + private static final int DEFAULT_CONSUMER_THREAD_NUM = 5; + + private static final int DISPATCH_TIMER_JOB_QUEUE_SIZE = Config.job_dispatch_timer_job_queue_size > 0 + ? Config.job_dispatch_timer_job_queue_size : DEFAULT_RING_BUFFER_SIZE; + + private static final int DISPATCH_TIMER_JOB_CONSUMER_THREAD_NUM = Config.job_dispatch_timer_job_thread_num > 0 + ? Config.job_dispatch_timer_job_thread_num : DEFAULT_CONSUMER_THREAD_NUM; + + private static final int DISPATCH_INSERT_THREAD_NUM = Config.job_insert_task_consumer_thread_num > 0 + ? Config.job_insert_task_consumer_thread_num : DEFAULT_RING_BUFFER_SIZE; + + private static final int DISPATCH_INSERT_TASK_QUEUE_SIZE = DEFAULT_RING_BUFFER_SIZE; + + + public void init() { + registerInsertDisruptor(); + //when all task queue is ready, dispatch task to registered task executor + registerDispatchDisruptor(); + } + + private void registerDispatchDisruptor() { + EventFactory>> dispatchEventFactory = TimerJobEvent.factory(); + ThreadFactory dispatchThreadFactory = new CustomThreadFactory("dispatch-task"); + WorkHandler[] dispatchTaskExecutorHandlers = new WorkHandler[DISPATCH_TIMER_JOB_CONSUMER_THREAD_NUM]; + for (int i = 0; i < DISPATCH_TIMER_JOB_CONSUMER_THREAD_NUM; i++) { + dispatchTaskExecutorHandlers[i] = new DispatchTaskHandler(this.disruptorMap); + } + EventTranslatorVararg>> eventTranslator = + (event, sequence, args) -> event.setJob((AbstractJob) args[0]); + this.dispatchDisruptor = new TaskDisruptor<>(dispatchEventFactory, DISPATCH_TIMER_JOB_QUEUE_SIZE, + dispatchThreadFactory, + new BlockingWaitStrategy(), dispatchTaskExecutorHandlers, eventTranslator); + } + + private void registerInsertDisruptor() { + EventFactory> insertEventFactory = ExecuteTaskEvent.factory(); + ThreadFactory insertTaskThreadFactory = new CustomThreadFactory("insert-task-execute"); + WorkHandler[] insertTaskExecutorHandlers = new WorkHandler[DISPATCH_INSERT_THREAD_NUM]; + for (int i = 0; i < DISPATCH_INSERT_THREAD_NUM; i++) { + insertTaskExecutorHandlers[i] = new DefaultTaskExecutorHandler(); + } + EventTranslatorVararg> eventTranslator = + (event, sequence, args) -> { + event.setTask((InsertTask) args[0]); + event.setJobConfig((JobExecutionConfiguration) args[1]); + }; + TaskDisruptor insertDisruptor = new TaskDisruptor<>(insertEventFactory, DISPATCH_INSERT_TASK_QUEUE_SIZE, + insertTaskThreadFactory, new BlockingWaitStrategy(), insertTaskExecutorHandlers, eventTranslator); + disruptorMap.put(JobType.INSERT, insertDisruptor); + } + + public void dispatchTimerJob(AbstractJob job) { + dispatchDisruptor.publishEvent(job); + } + + public void dispatchInstantTask(AbstractTask task, JobType jobType, + JobExecutionConfiguration jobExecutionConfiguration) { + disruptorMap.get(jobType).publishEvent(task, jobExecutionConfiguration); + } + + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/manager/TaskTokenManager.java b/fe/fe-core/src/main/java/org/apache/doris/job/manager/TaskTokenManager.java new file mode 100644 index 0000000000..51b8b6aeb3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/manager/TaskTokenManager.java @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.manager; + +import lombok.experimental.UtilityClass; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; + +/** + * TaskTokenManager is responsible for managing semaphore tokens for different jobs. + * It provides a method to acquire a semaphore token for a specific job ID with the given maximum concurrency. + * If a semaphore doesn't exist for the job ID, it creates a new one and adds it to the map. + */ +@Slf4j +@UtilityClass +public class TaskTokenManager { + + private static final Map taskTokenMap = new ConcurrentHashMap<>(16); + + /** + * Tries to acquire a semaphore token for the specified job ID with the given maximum concurrency. + * If a semaphore doesn't exist for the job ID, it creates a new one and adds it to the map. + * + * @param jobId the ID of the job + * @param maxConcurrent the maximum concurrency for the job + * @return the acquired semaphore + */ + public static Semaphore tryAcquire(long jobId, long maxConcurrent) { + Semaphore semaphore = taskTokenMap.computeIfAbsent(jobId, id -> new Semaphore((int) maxConcurrent)); + try { + semaphore.acquire(); + } catch (InterruptedException e) { + log.warn("Interrupted while acquiring semaphore for job id: {} ", jobId, e); + Thread.currentThread().interrupt(); + } + return semaphore; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java new file mode 100644 index 0000000000..a9cedc9994 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.scheduler; + +import org.apache.doris.common.CustomThreadFactory; +import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.job.base.AbstractJob; +import org.apache.doris.job.base.JobExecuteType; +import org.apache.doris.job.common.JobStatus; +import org.apache.doris.job.common.TaskType; +import org.apache.doris.job.disruptor.TaskDisruptor; +import org.apache.doris.job.exception.JobException; +import org.apache.doris.job.executor.TimerJobSchedulerTask; +import org.apache.doris.job.manager.TaskDisruptorGroupManager; +import org.apache.doris.job.task.AbstractTask; + +import io.netty.util.HashedWheelTimer; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +@Slf4j +public class JobScheduler> implements Closeable { + + /** + * scheduler tasks, it's used to scheduler job + */ + private HashedWheelTimer timerTaskScheduler; + + private TaskDisruptor timerJobDisruptor; + + private TaskDisruptorGroupManager taskDisruptorGroupManager; + + private long latestBatchSchedulerTimerTaskTimeMs = 0L; + + private static final long BATCH_SCHEDULER_INTERVAL_SECONDS = 60; + + private static final int HASHED_WHEEL_TIMER_TICKS_PER_WHEEL = 660; + + private final Map jobMap; + + public JobScheduler(Map jobMap) { + this.jobMap = jobMap; + } + + /** + * batch scheduler interval ms time + */ + private static final long BATCH_SCHEDULER_INTERVAL_MILLI_SECONDS = BATCH_SCHEDULER_INTERVAL_SECONDS * 1000L; + + public void start() { + timerTaskScheduler = new HashedWheelTimer(new CustomThreadFactory("timer-task-scheduler"), 1, + TimeUnit.SECONDS, HASHED_WHEEL_TIMER_TICKS_PER_WHEEL); + timerTaskScheduler.start(); + taskDisruptorGroupManager = new TaskDisruptorGroupManager(); + taskDisruptorGroupManager.init(); + this.timerJobDisruptor = taskDisruptorGroupManager.getDispatchDisruptor(); + latestBatchSchedulerTimerTaskTimeMs = System.currentTimeMillis(); + batchSchedulerTimerJob(); + cycleSystemSchedulerTasks(); + } + + /** + * We will cycle system scheduler tasks every 10 minutes. + * Jobs will be re-registered after the task is completed + */ + private void cycleSystemSchedulerTasks() { + log.info("re-register system scheduler timer tasks" + TimeUtils.longToTimeString(System.currentTimeMillis())); + timerTaskScheduler.newTimeout(timeout -> { + batchSchedulerTimerJob(); + cycleSystemSchedulerTasks(); + }, BATCH_SCHEDULER_INTERVAL_SECONDS, TimeUnit.SECONDS); + + } + + private void batchSchedulerTimerJob() { + executeTimerJobIdsWithinLastTenMinutesWindow(); + } + + public void scheduleOneJob(T job) throws JobException { + if (!job.getJobStatus().equals(JobStatus.RUNNING)) { + return; + } + if (!job.getJobConfig().checkIsTimerJob()) { + //manual job will not scheduler + if (JobExecuteType.MANUAL.equals(job.getJobConfig().getExecuteType())) { + return; + } + //todo skip streaming job,improve in the future + if (JobExecuteType.INSTANT.equals(job.getJobConfig().getExecuteType())) { + schedulerInstantJob(job, TaskType.SCHEDULED); + } + } + //if it's timer job and trigger last window already start, we will scheduler it immediately + cycleTimerJobScheduler(job); + } + + @Override + public void close() throws IOException { + //todo implement this later + } + + + private void cycleTimerJobScheduler(T job) { + List delaySeconds = job.getJobConfig().getTriggerDelayTimes(System.currentTimeMillis(), + System.currentTimeMillis(), latestBatchSchedulerTimerTaskTimeMs); + if (CollectionUtils.isNotEmpty(delaySeconds)) { + delaySeconds.forEach(delaySecond -> { + TimerJobSchedulerTask timerJobSchedulerTask = new TimerJobSchedulerTask<>(timerJobDisruptor, job); + timerTaskScheduler.newTimeout(timerJobSchedulerTask, delaySecond, TimeUnit.SECONDS); + }); + } + } + + + public void schedulerInstantJob(T job, TaskType taskType) throws JobException { + if (!job.getJobStatus().equals(JobStatus.RUNNING)) { + throw new JobException("job is not running,job id is %d", job.getJobId()); + } + if (!job.isReadyForScheduling()) { + log.info("job is not ready for scheduling,job id is {}", job.getJobId()); + return; + } + List tasks = job.createTasks(taskType); + if (CollectionUtils.isEmpty(tasks)) { + if (job.getJobConfig().getExecuteType().equals(JobExecuteType.INSTANT)) { + job.setJobStatus(JobStatus.FINISHED); + } + return; + } + tasks.forEach(task -> taskDisruptorGroupManager.dispatchInstantTask(task, job.getJobType(), + job.getJobConfig())); + + } + + /** + * We will get the task in the next time window, and then hand it over to the time wheel for timing trigger + */ + private void executeTimerJobIdsWithinLastTenMinutesWindow() { + if (jobMap.isEmpty()) { + return; + } + if (latestBatchSchedulerTimerTaskTimeMs < System.currentTimeMillis()) { + this.latestBatchSchedulerTimerTaskTimeMs = System.currentTimeMillis(); + } + this.latestBatchSchedulerTimerTaskTimeMs += BATCH_SCHEDULER_INTERVAL_MILLI_SECONDS; + for (Map.Entry entry : jobMap.entrySet()) { + T job = entry.getValue(); + if (!job.getJobConfig().checkIsTimerJob()) { + continue; + } + cycleTimerJobScheduler(job); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/task/AbstractTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/task/AbstractTask.java new file mode 100644 index 0000000000..7fc666339a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/task/AbstractTask.java @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.task; + +import org.apache.doris.catalog.Env; +import org.apache.doris.job.base.Job; +import org.apache.doris.job.common.TaskStatus; +import org.apache.doris.job.common.TaskType; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +@Data +@Slf4j +public abstract class AbstractTask implements Task { + + private Long jobId; + + private Long taskId; + + private TaskStatus status; + + private Long createTimeMs; + + private Long startTimeMs; + + private Long finishTimeMs; + + private TaskType taskType; + + @Override + public void onFail(String msg) { + if (!isCallable()) { + return; + } + Env.getCurrentEnv().getJobManager().getJob(jobId).onTaskFail(this); + status = TaskStatus.FAILD; + } + + @Override + public void onFail() { + setFinishTimeMs(System.currentTimeMillis()); + if (!isCallable()) { + return; + } + Job job = Env.getCurrentEnv().getJobManager().getJob(getJobId()); + job.onTaskFail(this); + } + + private boolean isCallable() { + if (status.equals(TaskStatus.CANCEL)) { + return false; + } + if (null != Env.getCurrentEnv().getJobManager().getJob(jobId)) { + return true; + } + return false; + } + + @Override + public void onSuccess() { + status = TaskStatus.SUCCESS; + setFinishTimeMs(System.currentTimeMillis()); + if (!isCallable()) { + return; + } + Job job = Env.getCurrentEnv().getJobManager().getJob(getJobId()); + if (null == job) { + log.info("job is null, job id is {}", jobId); + return; + } + job.onTaskSuccess(this); + } + + @Override + public void cancel() { + status = TaskStatus.CANCEL; + } + + @Override + public void before() { + status = TaskStatus.RUNNING; + setStartTimeMs(System.currentTimeMillis()); + } + + public void runTask() { + try { + before(); + run(); + onSuccess(); + } catch (Exception e) { + onFail(); + log.warn("execute task error, job id is {},task id is {}", jobId, taskId, e); + } + } + + public boolean isCancelled() { + return status.equals(TaskStatus.CANCEL); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/task/Task.java b/fe/fe-core/src/main/java/org/apache/doris/job/task/Task.java new file mode 100644 index 0000000000..350c562f7e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/job/task/Task.java @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.task; + +/** + * The Task interface represents a task that can be executed and managed by a scheduler. + * All extension tasks must implement this interface. + * The methods defined in this interface are automatically called by the scheduler before and after the execution + * of the run method. + */ +public interface Task { + + /** + * This method is called before the task is executed. + * Implementations can use this method to perform any necessary setup or initialization. + */ + void before(); + + /** + * This method contains the main logic of the task. + * Implementations should define the specific actions to be performed by the task. + */ + void run(); + + /** + * This method is called when the task fails to execute successfully. + * Implementations can use this method to handle any failure scenarios. + */ + void onFail(); + + /** + * This method is called when the task fails to execute successfully, with an additional error message. + * Implementations can use this method to handle any failure scenarios and provide a custom error message. + * + * @param msg The error message associated with the failure. + */ + void onFail(String msg); + + /** + * This method is called when the task executes successfully. + * Implementations can use this method to handle successful execution scenarios. + */ + void onSuccess(); + + /** + * This method is called to cancel the execution of the task. + * Implementations should define the necessary steps to cancel the task. + */ + void cancel(); + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java index ba2fe4becf..162e8b3cc8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java @@ -44,6 +44,7 @@ import org.apache.doris.datasource.InitCatalogLog; import org.apache.doris.datasource.InitDatabaseLog; import org.apache.doris.datasource.InitTableLog; import org.apache.doris.ha.MasterInfo; +import org.apache.doris.job.base.AbstractJob; import org.apache.doris.journal.bdbje.Timestamp; import org.apache.doris.load.DeleteInfo; import org.apache.doris.load.ExportJob; @@ -117,8 +118,6 @@ import org.apache.doris.policy.DropPolicyLog; import org.apache.doris.policy.Policy; import org.apache.doris.policy.StoragePolicy; import org.apache.doris.resource.workloadgroup.WorkloadGroup; -import org.apache.doris.scheduler.job.Job; -import org.apache.doris.scheduler.job.JobTask; import org.apache.doris.statistics.AnalysisInfo; import org.apache.doris.statistics.TableStatsMeta; import org.apache.doris.system.Backend; @@ -538,16 +537,14 @@ public class JournalEntity implements Writable { case OperationType.OP_UPDATE_SCHEDULER_JOB: case OperationType.OP_DELETE_SCHEDULER_JOB: case OperationType.OP_CREATE_SCHEDULER_JOB: { - Job job = Job.readFields(in); + AbstractJob job = AbstractJob.readFields(in); data = job; isRead = true; break; } case OperationType.OP_CREATE_SCHEDULER_TASK: case OperationType.OP_DELETE_SCHEDULER_TASK: { - JobTask task = JobTask.readFields(in); - data = task; - isRead = true; + //todo improve break; } case OperationType.OP_CREATE_LOAD_JOB: { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSchedule.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSchedule.java index 2f4aa5f96c..17123a3235 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSchedule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSchedule.java @@ -18,8 +18,8 @@ package org.apache.doris.mtmv; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.job.common.IntervalUnit; import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.scheduler.common.IntervalUnit; import com.google.gson.annotations.SerializedName; import org.apache.commons.lang3.StringUtils; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 232334ca37..18333878f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -28,6 +28,7 @@ import org.apache.doris.catalog.KeysType; import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; import org.apache.doris.common.Pair; +import org.apache.doris.job.common.IntervalUnit; import org.apache.doris.load.loadv2.LoadTask; import org.apache.doris.mtmv.MTMVRefreshEnum.BuildMode; import org.apache.doris.mtmv.MTMVRefreshEnum.RefreshMethod; @@ -375,7 +376,6 @@ import org.apache.doris.policy.FilterType; import org.apache.doris.policy.PolicyTypeEnum; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SqlModeHelper; -import org.apache.doris.scheduler.common.IntervalUnit; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 877b6cc18e..f34d0c2c69 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -52,6 +52,7 @@ import org.apache.doris.datasource.ExternalObjectLog; import org.apache.doris.datasource.InitCatalogLog; import org.apache.doris.datasource.InitDatabaseLog; import org.apache.doris.ha.MasterInfo; +import org.apache.doris.job.base.AbstractJob; import org.apache.doris.journal.Journal; import org.apache.doris.journal.JournalCursor; import org.apache.doris.journal.JournalEntity; @@ -78,8 +79,6 @@ import org.apache.doris.policy.DropPolicyLog; import org.apache.doris.policy.Policy; import org.apache.doris.policy.StoragePolicy; import org.apache.doris.resource.workloadgroup.WorkloadGroup; -import org.apache.doris.scheduler.job.Job; -import org.apache.doris.scheduler.job.JobTask; import org.apache.doris.statistics.AnalysisInfo; import org.apache.doris.statistics.AnalysisManager; import org.apache.doris.statistics.TableStatsMeta; @@ -662,21 +661,21 @@ public class EditLog { break; } case OperationType.OP_CREATE_SCHEDULER_JOB: { - Job job = (Job) journal.getData(); - Env.getCurrentEnv().getAsyncJobManager().replayCreateJob(job); + AbstractJob job = (AbstractJob) journal.getData(); + Env.getCurrentEnv().getJobManager().replayCreateJob(job); break; } case OperationType.OP_UPDATE_SCHEDULER_JOB: { - Job job = (Job) journal.getData(); - Env.getCurrentEnv().getAsyncJobManager().replayUpdateJob(job); + AbstractJob job = (AbstractJob) journal.getData(); + Env.getCurrentEnv().getJobManager().replayUpdateJob(job); break; } case OperationType.OP_DELETE_SCHEDULER_JOB: { - Job job = (Job) journal.getData(); - Env.getCurrentEnv().getAsyncJobManager().replayDeleteJob(job); + AbstractJob job = (AbstractJob) journal.getData(); + Env.getCurrentEnv().getJobManager().replayDeleteJob(job); break; } - case OperationType.OP_CREATE_SCHEDULER_TASK: { + /*case OperationType.OP_CREATE_SCHEDULER_TASK: { JobTask task = (JobTask) journal.getData(); Env.getCurrentEnv().getJobTaskManager().replayCreateTask(task); break; @@ -685,7 +684,7 @@ public class EditLog { JobTask task = (JobTask) journal.getData(); Env.getCurrentEnv().getJobTaskManager().replayDeleteTask(task); break; - } + }*/ case OperationType.OP_CHANGE_ROUTINE_LOAD_JOB: { RoutineLoadOperation operation = (RoutineLoadOperation) journal.getData(); Env.getCurrentEnv().getRoutineLoadManager().replayChangeRoutineLoadJob(operation); @@ -1603,23 +1602,15 @@ public class EditLog { logEdit(OperationType.OP_CREATE_ROUTINE_LOAD_JOB, routineLoadJob); } - public void logCreateJob(Job job) { + public void logCreateJob(AbstractJob job) { logEdit(OperationType.OP_CREATE_SCHEDULER_JOB, job); } - public void logUpdateJob(Job job) { + public void logUpdateJob(AbstractJob job) { logEdit(OperationType.OP_UPDATE_SCHEDULER_JOB, job); } - public void logCreateJobTask(JobTask jobTask) { - logEdit(OperationType.OP_CREATE_SCHEDULER_TASK, jobTask); - } - - public void logDeleteJobTask(JobTask jobTask) { - logEdit(OperationType.OP_DELETE_SCHEDULER_TASK, jobTask); - } - - public void logDeleteJob(Job job) { + public void logDeleteJob(AbstractJob job) { logEdit(OperationType.OP_DELETE_SCHEDULER_JOB, job); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index 9f4bcd86f7..8b6f85d511 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -77,6 +77,8 @@ import org.apache.doris.datasource.jdbc.JdbcExternalCatalog; import org.apache.doris.datasource.paimon.PaimonExternalCatalog; import org.apache.doris.datasource.paimon.PaimonFileExternalCatalog; import org.apache.doris.datasource.paimon.PaimonHMSExternalCatalog; +import org.apache.doris.job.base.AbstractJob; +import org.apache.doris.job.extensions.insert.InsertJob; import org.apache.doris.load.loadv2.LoadJob.LoadJobStateUpdateInfo; import org.apache.doris.load.loadv2.SparkLoadJob.SparkLoadJobStateUpdateInfo; import org.apache.doris.load.routineload.AbstractDataSourceProperties; @@ -86,8 +88,6 @@ import org.apache.doris.load.sync.canal.CanalSyncJob; import org.apache.doris.policy.Policy; import org.apache.doris.policy.RowPolicy; import org.apache.doris.policy.StoragePolicy; -import org.apache.doris.scheduler.executor.JobExecutor; -import org.apache.doris.scheduler.executor.SqlJobExecutor; import org.apache.doris.system.BackendHbResponse; import org.apache.doris.system.BrokerHbResponse; import org.apache.doris.system.FrontendHbResponse; @@ -221,10 +221,10 @@ public class GsonUtils { RuntimeTypeAdapterFactory.of( AbstractDataSourceProperties.class, "clazz") .registerSubtype(KafkaDataSourceProperties.class, KafkaDataSourceProperties.class.getSimpleName()); - private static RuntimeTypeAdapterFactory jobExecutorRuntimeTypeAdapterFactory = + private static RuntimeTypeAdapterFactory jobExecutorRuntimeTypeAdapterFactory = RuntimeTypeAdapterFactory.of( - JobExecutor.class, "clazz") - .registerSubtype(SqlJobExecutor.class, SqlJobExecutor.class.getSimpleName()); + AbstractJob.class, "clazz") + .registerSubtype(InsertJob.class, InsertJob.class.getSimpleName()); private static RuntimeTypeAdapterFactory dbTypeAdapterFactory = RuntimeTypeAdapterFactory.of( DatabaseIf.class, "clazz") diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java index 715cd26796..6ed00de22b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java @@ -232,10 +232,6 @@ public class MetaPersistMethod { Env.class.getDeclaredMethod("saveAsyncJobManager", CountingDataOutputStream.class, long.class); break; case "JobTaskManager": - metaPersistMethod.readMethod = - Env.class.getDeclaredMethod("loadJobTaskManager", DataInputStream.class, long.class); - metaPersistMethod.writeMethod = - Env.class.getDeclaredMethod("saveJobTaskManager", CountingDataOutputStream.class, long.class); break; default: break; diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java index f9758698c8..f6161629d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java @@ -39,7 +39,7 @@ public class PersistMetaModules { "globalVariable", "cluster", "broker", "resources", "exportJob", "syncJob", "backupHandler", "paloAuth", "transactionState", "colocateTableIndex", "routineLoadJobs", "loadJobV2", "smallFiles", "plugins", "deleteHandler", "sqlBlockRule", "policy", "globalFunction", "workloadGroups", - "binlogs", "resourceGroups", "AnalysisMgrV2", "AsyncJobManager", "JobTaskManager"); + "binlogs", "resourceGroups", "AnalysisMgrV2", "AsyncJobManager"); // Modules in this list is deprecated and will not be saved in meta file. (also should not be in MODULE_NAMES) public static final ImmutableList DEPRECATED_MODULE_NAMES = ImmutableList.of( diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java index 483fda4ab5..2013216742 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java @@ -121,9 +121,9 @@ import org.apache.doris.catalog.EncryptKeyHelper; import org.apache.doris.catalog.Env; import org.apache.doris.common.DdlException; import org.apache.doris.common.util.ProfileManager; +import org.apache.doris.job.common.JobStatus; import org.apache.doris.load.sync.SyncJobManager; import org.apache.doris.persist.CleanQueryStatsInfo; -import org.apache.doris.scheduler.constants.JobCategory; import org.apache.doris.statistics.StatisticsRepository; import org.apache.logging.log4j.LogManager; @@ -184,16 +184,16 @@ public class DdlExecutor { } else if (ddlStmt instanceof AlterRoutineLoadStmt) { env.getRoutineLoadManager().alterRoutineLoadJob((AlterRoutineLoadStmt) ddlStmt); } else if (ddlStmt instanceof CreateJobStmt) { - env.getJobRegister().registerJob((((CreateJobStmt) ddlStmt).getJob())); + env.getJobManager().registerJob(((CreateJobStmt) ddlStmt).getJobInstance()); } else if (ddlStmt instanceof StopJobStmt) { StopJobStmt stmt = (StopJobStmt) ddlStmt; - env.getJobRegister().stopJob(stmt.getDbFullName(), stmt.getName(), JobCategory.SQL); + env.getJobManager().unregisterJob(stmt.getDbFullName(), stmt.getName()); } else if (ddlStmt instanceof PauseJobStmt) { PauseJobStmt stmt = (PauseJobStmt) ddlStmt; - env.getJobRegister().pauseJob(stmt.getDbFullName(), stmt.getName(), JobCategory.SQL); + env.getJobManager().alterJobStatus(stmt.getDbFullName(), stmt.getName(), JobStatus.PAUSED); } else if (ddlStmt instanceof ResumeJobStmt) { ResumeJobStmt stmt = (ResumeJobStmt) ddlStmt; - env.getJobRegister().resumeJob(stmt.getDbFullName(), stmt.getName(), JobCategory.SQL); + env.getJobManager().alterJobStatus(stmt.getDbFullName(), stmt.getName(), JobStatus.RUNNING); } else if (ddlStmt instanceof CreateUserStmt) { CreateUserStmt stmt = (CreateUserStmt) ddlStmt; env.getAuth().createUser(stmt); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 2aadab7885..1ddd91083f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -189,8 +189,6 @@ import org.apache.doris.load.LoadJob.JobState; import org.apache.doris.load.loadv2.LoadManager; import org.apache.doris.load.routineload.RoutineLoadJob; import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.scheduler.job.Job; -import org.apache.doris.scheduler.job.JobTask; import org.apache.doris.statistics.AnalysisInfo; import org.apache.doris.statistics.ColumnStatistic; import org.apache.doris.statistics.Histogram; @@ -429,7 +427,7 @@ public class ShowExecutor { } else if (stmt instanceof ShowJobStmt) { handleShowJob(); } else if (stmt instanceof ShowJobTaskStmt) { - handleShowJobTask(); + //handleShowJobTask(); } else if (stmt instanceof ShowConvertLSCStmt) { handleShowConvertLSC(); } else { @@ -1418,7 +1416,7 @@ public class ShowExecutor { resultSet = new ShowResultSet(showWarningsStmt.getMetaData(), rows); } - private void handleShowJobTask() { + /*private void handleShowJobTask() { ShowJobTaskStmt showJobTaskStmt = (ShowJobTaskStmt) stmt; List> rows = Lists.newArrayList(); List jobs = Env.getCurrentEnv().getJobRegister() @@ -1439,21 +1437,15 @@ public class ShowExecutor { rows.add(jobTask.getShowInfo(job.getJobName())); } resultSet = new ShowResultSet(showJobTaskStmt.getMetaData(), rows); - } + }*/ private void handleShowJob() throws AnalysisException { ShowJobStmt showJobStmt = (ShowJobStmt) stmt; List> rows = Lists.newArrayList(); // if job exists - List jobList; - PatternMatcher matcher = null; - if (showJobStmt.getPattern() != null) { - matcher = PatternMatcherWrapper.createMysqlPattern(showJobStmt.getPattern(), - CaseSensibility.JOB.getCaseSensibility()); - } - jobList = Env.getCurrentEnv().getJobRegister() - .getJobs(showJobStmt.getDbFullName(), showJobStmt.getName(), showJobStmt.getJobCategory(), - matcher); + List jobList; + jobList = Env.getCurrentEnv().getJobManager() + .queryJobs(showJobStmt.getDbFullName(), showJobStmt.getName()); if (jobList.isEmpty()) { resultSet = new ShowResultSet(showJobStmt.getMetaData(), rows); @@ -1461,15 +1453,9 @@ public class ShowExecutor { } // check auth - for (Job job : jobList) { - if (!Env.getCurrentEnv().getAccessManager() - .checkDbPriv(ConnectContext.get(), job.getDbName(), PrivPredicate.SHOW)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DBACCESS_DENIED_ERROR, - ConnectContext.get().getQualifiedUser(), job.getDbName()); - } - } - for (Job job : jobList) { - rows.add(job.getShowInfo()); + + for (org.apache.doris.job.base.AbstractJob job : jobList) { + rows.add(job.getCommonShowInfo()); } resultSet = new ShowResultSet(showJobStmt.getMetaData(), rows); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/constants/JobCategory.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/constants/JobCategory.java deleted file mode 100644 index 8578314721..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/constants/JobCategory.java +++ /dev/null @@ -1,58 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.scheduler.constants; - -import lombok.Getter; - -/** - * The job category is used to distinguish different types of jobs. - */ -public enum JobCategory { - COMMON(1, "common", true), - SQL(2, "sql", true), - MTMV(3, "mtmv", false), - ; - - @Getter - private int code; - - @Getter - private String name; - - /** - * if the job is persistent, it will be saved to the metadata store. - * if the job is not persistent, it will not be saved to the memory. - */ - @Getter - private boolean persistent; - - JobCategory(int code, String name, boolean persistent) { - this.code = code; - this.name = name; - this.persistent = persistent; - } - - public static JobCategory getJobCategoryByName(String name) { - for (JobCategory jobCategory : JobCategory.values()) { - if (jobCategory.name.equalsIgnoreCase(name)) { - return jobCategory; - } - } - throw new IllegalArgumentException("Unknown job category name: " + name); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TaskDisruptor.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TaskDisruptor.java index 0e3f8e618d..a20f772eda 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TaskDisruptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TaskDisruptor.java @@ -19,7 +19,6 @@ package org.apache.doris.scheduler.disruptor; import org.apache.doris.common.Config; import org.apache.doris.scheduler.constants.TaskType; -import org.apache.doris.scheduler.manager.TimerJobManager; import org.apache.doris.scheduler.manager.TransientTaskManager; import com.lmax.disruptor.BlockingWaitStrategy; @@ -48,8 +47,6 @@ import java.util.concurrent.TimeUnit; public class TaskDisruptor implements Closeable { private Disruptor disruptor; - - private TimerJobManager timerJobManager; private TransientTaskManager transientTaskManager; private static final int DEFAULT_RING_BUFFER_SIZE = Config.async_task_queen_size; @@ -77,8 +74,7 @@ public class TaskDisruptor implements Closeable { event.setTaskType(taskType); }; - public TaskDisruptor(TimerJobManager timerJobManager, TransientTaskManager transientTaskManager) { - this.timerJobManager = timerJobManager; + public TaskDisruptor(TransientTaskManager transientTaskManager) { this.transientTaskManager = transientTaskManager; } @@ -88,7 +84,7 @@ public class TaskDisruptor implements Closeable { ProducerType.SINGLE, new BlockingWaitStrategy()); WorkHandler[] workers = new TaskHandler[consumerThreadCount]; for (int i = 0; i < consumerThreadCount; i++) { - workers[i] = new TaskHandler(timerJobManager, transientTaskManager); + workers[i] = new TaskHandler(transientTaskManager); } disruptor.handleEventsWithWorkerPool(workers); disruptor.start(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TaskHandler.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TaskHandler.java index 6005fa1bd4..6a0b9f92c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TaskHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TaskHandler.java @@ -17,16 +17,8 @@ package org.apache.doris.scheduler.disruptor; -import org.apache.doris.catalog.Env; -import org.apache.doris.persist.gson.GsonUtils; -import org.apache.doris.scheduler.constants.JobType; import org.apache.doris.scheduler.exception.JobException; import org.apache.doris.scheduler.executor.TransientTaskExecutor; -import org.apache.doris.scheduler.job.ExecutorResult; -import org.apache.doris.scheduler.job.Job; -import org.apache.doris.scheduler.job.JobTask; -import org.apache.doris.scheduler.manager.JobTaskManager; -import org.apache.doris.scheduler.manager.TimerJobManager; import org.apache.doris.scheduler.manager.TransientTaskManager; import com.lmax.disruptor.WorkHandler; @@ -42,22 +34,10 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class TaskHandler implements WorkHandler { - /** - * The event job manager used to retrieve and execute event jobs. - */ - private TimerJobManager timerJobManager; private TransientTaskManager transientTaskManager; - private JobTaskManager jobTaskManager; - - /** - * Constructs a new {@link TaskHandler} instance with the specified event job manager. - * - * @param timerJobManager The event job manager used to retrieve and execute event jobs. - */ - public TaskHandler(TimerJobManager timerJobManager, TransientTaskManager transientTaskManager) { - this.timerJobManager = timerJobManager; + public TaskHandler(TransientTaskManager transientTaskManager) { this.transientTaskManager = transientTaskManager; } @@ -71,10 +51,6 @@ public class TaskHandler implements WorkHandler { @Override public void onEvent(TaskEvent event) { switch (event.getTaskType()) { - case SCHEDULER_JOB_TASK: - case MANUAL_JOB_TASK: - onTimerJobTaskHandle(event); - break; case TRANSIENT_TASK: onTransientTaskHandle(event); break; @@ -84,71 +60,6 @@ public class TaskHandler implements WorkHandler { } } - /** - * Processes an event task by retrieving the associated event job and executing it if it is running. - * - * @param taskEvent The event task to be processed. - */ - @SuppressWarnings("checkstyle:UnusedLocalVariable") - public void onTimerJobTaskHandle(TaskEvent taskEvent) { - long jobId = taskEvent.getId(); - long taskId = taskEvent.getTaskId(); - JobTask jobTask = jobTaskManager.pollPrepareTaskByTaskId(jobId, taskId); - if (jobTask == null) { - log.warn("jobTask is null, maybe it's cancel, jobId: {}, taskId: {}", jobId, taskId); - return; - } - Job job = timerJobManager.getJob(jobId); - if (job == null) { - log.info("job is null, jobId: {}", jobId); - return; - } - if (!job.isRunning()) { - log.info("job is not running, eventJobId: {}", jobId); - return; - } - log.debug("job is running, eventJobId: {}", jobId); - - - try { - jobTask.setStartTimeMs(System.currentTimeMillis()); - ExecutorResult result = job.getExecutor().execute(job, jobTask.getContextData()); - job.setLatestCompleteExecuteTimeMs(System.currentTimeMillis()); - if (job.getJobType().equals(JobType.RECURRING)) { - updateJobStatusIfPastEndTime(job); - } else { - // one time job should be finished after execute - updateOnceTimeJobStatus(job); - } - if (null == result) { - log.warn("Job execute failed, jobId: {}, result is null", jobId); - jobTask.setErrorMsg("Job execute failed, result is null"); - jobTask.setIsSuccessful(false); - timerJobManager.setJobLatestStatus(jobId, false); - return; - } - String resultStr = GsonUtils.GSON.toJson(result.getResult()); - jobTask.setExecuteResult(resultStr); - jobTask.setIsSuccessful(result.isSuccess()); - if (!result.isSuccess()) { - log.warn("Job execute failed, jobId: {}, msg : {}", jobId, result.getExecutorSql()); - jobTask.setErrorMsg(result.getErrorMsg()); - } - jobTask.setExecuteSql(result.getExecutorSql()); - } catch (Exception e) { - log.warn("Job execute failed, jobId: {}, msg : {}", jobId, e.getMessage()); - jobTask.setErrorMsg(e.getMessage()); - jobTask.setIsSuccessful(false); - } - jobTask.setEndTimeMs(System.currentTimeMillis()); - if (null == jobTaskManager) { - jobTaskManager = Env.getCurrentEnv().getJobTaskManager(); - } - boolean isPersistent = job.getJobCategory().isPersistent(); - jobTaskManager.addJobTask(jobTask, isPersistent); - timerJobManager.setJobLatestStatus(jobId, jobTask.getIsSuccessful()); - } - public void onTransientTaskHandle(TaskEvent taskEvent) { Long taskId = taskEvent.getId(); TransientTaskExecutor taskExecutor = transientTaskManager.getMemoryTaskExecutor(taskId); @@ -164,18 +75,4 @@ public class TaskHandler implements WorkHandler { } } - private void updateJobStatusIfPastEndTime(Job job) { - if (job.isExpired()) { - timerJobManager.finishJob(job.getJobId()); - } - } - - private void updateOnceTimeJobStatus(Job job) { - if (job.getJobType().equals(JobType.STREAMING)) { - timerJobManager.putOneJobToQueen(job.getJobId()); - return; - } - job.finish(); - } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/executor/AbstractJobExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/executor/AbstractJobExecutor.java deleted file mode 100644 index 7a70c963bd..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/executor/AbstractJobExecutor.java +++ /dev/null @@ -1,54 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.scheduler.executor; - -import org.apache.doris.analysis.UserIdentity; -import org.apache.doris.catalog.Env; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.scheduler.job.Job; -import org.apache.doris.thrift.TUniqueId; - -import lombok.Getter; - -import java.util.UUID; - -@Getter -public abstract class AbstractJobExecutor implements JobExecutor { - - protected ConnectContext createContext(Job job) { - ConnectContext ctx = new ConnectContext(); - ctx.setEnv(Env.getCurrentEnv()); - ctx.setCluster(ClusterNamespace.getClusterNameFromFullName(job.getDbName())); - ctx.setDatabase(job.getDbName()); - ctx.setQualifiedUser(job.getUser()); - ctx.setCurrentUserIdentity(UserIdentity.createAnalyzedUserIdentWithIp(job.getUser(), "%")); - ctx.getState().reset(); - ctx.setThreadLocalInfo(); - return ctx; - } - - protected String generateTaskId() { - return UUID.randomUUID().toString(); - } - - protected TUniqueId generateQueryId(String taskIdString) { - UUID taskId = UUID.fromString(taskIdString); - return new TUniqueId(taskId.getMostSignificantBits(), taskId.getLeastSignificantBits()); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/executor/JobExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/executor/JobExecutor.java deleted file mode 100644 index 40aebc8f6a..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/executor/JobExecutor.java +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.scheduler.executor; - -import org.apache.doris.scheduler.exception.JobException; -import org.apache.doris.scheduler.job.ExecutorResult; -import org.apache.doris.scheduler.job.Job; - -/** - * This interface represents a callback for an event registration. All event registrations - * must implement this interface to provide an execution method. - * We will persist JobExecutor in the database, and then execute it when the scheduler starts. - * We use Gson to serialize and deserialize JobExecutor. so the implementation of JobExecutor needs to be serializable. - * You can see @org.apache.doris.persist.gson.GsonUtils.java for details.When you implement JobExecutor,pls make sure - * you can serialize and deserialize it. - */ -@FunctionalInterface -public interface JobExecutor { - - /** - * Executes the event job and returns the result. - * Exceptions will be caught internally, so there is no need to define or throw them separately. - * - * @param job The event job to execute. - * @param dataContext The data context of the event job. if you need to pass parameters to the event job, - * you can use it. - * @return The result of the event job execution. - */ - ExecutorResult execute(Job job, C dataContext) throws JobException; -} - diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/executor/SqlJobExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/executor/SqlJobExecutor.java deleted file mode 100644 index 546eac9a76..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/executor/SqlJobExecutor.java +++ /dev/null @@ -1,79 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.scheduler.executor; - -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.StmtExecutor; -import org.apache.doris.scheduler.exception.JobException; -import org.apache.doris.scheduler.job.ExecutorResult; -import org.apache.doris.scheduler.job.Job; -import org.apache.doris.thrift.TUniqueId; - -import com.google.gson.annotations.SerializedName; -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -import java.util.Map; - -/** - * we use this executor to execute sql job - */ -@Getter -@Slf4j -public class SqlJobExecutor extends AbstractJobExecutor> { - - @Setter - @SerializedName(value = "sql") - private String sql; - - public SqlJobExecutor(String sql) { - this.sql = sql; - } - - @Override - public ExecutorResult execute(Job job, Map dataContext) throws JobException { - ConnectContext ctx = createContext(job); - String taskIdString = generateTaskId(); - TUniqueId queryId = generateQueryId(taskIdString); - try { - StmtExecutor executor = new StmtExecutor(ctx, sql); - executor.execute(queryId); - String result = convertExecuteResult(ctx, taskIdString); - return new ExecutorResult<>(result, true, null, sql); - } catch (Exception e) { - log.warn("execute sql job failed, job id :{}, sql: {}, error: {}", job.getJobId(), sql, e); - return new ExecutorResult<>(null, false, e.getMessage(), sql); - } - - } - - private String convertExecuteResult(ConnectContext ctx, String queryId) throws JobException { - if (null == ctx.getState()) { - throw new JobException("execute sql job failed, sql: " + sql + ", error: response state is null"); - } - if (null != ctx.getState().getErrorCode()) { - throw new JobException("error code: " + ctx.getState().getErrorCode() + ", error msg: " - + ctx.getState().getErrorMessage()); - } - - return "queryId:" + queryId + ",affectedRows : " + ctx.getState().getAffectedRows() + ", warningRows: " - + ctx.getState().getWarningRows() + ",infoMsg" + ctx.getState().getInfoMessage(); - } - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/job/Job.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/job/Job.java deleted file mode 100644 index 7f3fd0884b..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/job/Job.java +++ /dev/null @@ -1,292 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.scheduler.job; - -import org.apache.doris.catalog.Env; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.io.Text; -import org.apache.doris.common.io.Writable; -import org.apache.doris.common.util.TimeUtils; -import org.apache.doris.persist.gson.GsonUtils; -import org.apache.doris.scheduler.common.IntervalUnit; -import org.apache.doris.scheduler.constants.JobCategory; -import org.apache.doris.scheduler.constants.JobStatus; -import org.apache.doris.scheduler.constants.JobType; -import org.apache.doris.scheduler.executor.JobExecutor; - -import com.google.common.collect.Lists; -import com.google.gson.annotations.SerializedName; -import lombok.Data; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.List; - -/** - * Job is the core of the scheduler module, which is used to store the Job information of the job module. - * We can use the job to uniquely identify a Job. - * The jobName is used to identify the job, which is not unique. - * The jobStatus is used to identify the status of the Job, which is used to control the execution of the - * job. - */ -@Data -public class Job implements Writable { - - public Job(String jobName, Long intervalMilliSeconds, Long startTimeMs, Long endTimeMs, - JobExecutor executor) { - this.jobName = jobName; - this.executor = executor; - this.intervalMs = intervalMilliSeconds; - this.startTimeMs = null == startTimeMs ? 0L : startTimeMs; - this.endTimeMs = null == endTimeMs ? 0L : endTimeMs; - this.jobStatus = JobStatus.RUNNING; - this.jobId = Env.getCurrentEnv().getNextId(); - } - - public Job() { - this.jobId = Env.getCurrentEnv().getNextId(); - } - - @SerializedName("jobId") - private Long jobId; - - @SerializedName("jobName") - private String jobName; - - @SerializedName("dbName") - private String dbName; - - /** - * The status of the job, which is used to control the execution of the job. - * - * @see JobStatus - */ - @SerializedName("jobStatus") - private JobStatus jobStatus; - - @SerializedName("jobType") - private JobType jobType = JobType.RECURRING; - - /** - * The executor of the job. - * - * @see JobExecutor - */ - @SerializedName("executor") - private JobExecutor executor; - @SerializedName("baseName") - private String baseName; - - @SerializedName("user") - private String user; - - @SerializedName("intervalMs") - private Long intervalMs = 0L; - @SerializedName("startTimeMs") - private Long startTimeMs = 0L; - - @SerializedName("endTimeMs") - private Long endTimeMs = 0L; - - @SerializedName("timezone") - private String timezone; - - @SerializedName("jobCategory") - private JobCategory jobCategory; - - - @SerializedName("latestStartExecuteTimeMs") - private Long latestStartExecuteTimeMs = 0L; - @SerializedName("latestCompleteExecuteTimeMs") - private Long latestCompleteExecuteTimeMs = 0L; - - @SerializedName("intervalUnit") - private IntervalUnit intervalUnit; - @SerializedName("originInterval") - private Long originInterval; - @SerializedName("nextExecuteTimeMs") - private Long nextExecuteTimeMs = 0L; - - @SerializedName("createTimeMs") - private Long createTimeMs = System.currentTimeMillis(); - - private Boolean lastExecuteTaskStatus; - - @SerializedName("comment") - private String comment; - - @SerializedName("errMsg") - private String errMsg; - - /** - * if we want to start the job immediately, we can set this flag to true. - * The default value is false. - * when we set this flag to true, the start time will be set to current time. - * we don't need to serialize this field. - */ - private boolean immediatelyStart = false; - - public boolean isRunning() { - return jobStatus == JobStatus.RUNNING; - } - - public boolean isStopped() { - return jobStatus == JobStatus.STOPPED; - } - - public boolean isFinished() { - return jobStatus == JobStatus.FINISHED; - } - - public boolean isExpired(long nextExecuteTimestamp) { - if (endTimeMs == 0L) { - return false; - } - return nextExecuteTimestamp > endTimeMs; - } - - public boolean isTaskTimeExceeded() { - if (endTimeMs == 0L) { - return false; - } - return System.currentTimeMillis() >= endTimeMs || nextExecuteTimeMs > endTimeMs; - } - - public boolean isExpired() { - if (endTimeMs == 0L) { - return false; - } - return System.currentTimeMillis() >= endTimeMs; - } - - public Long getExecuteTimestampAndGeneratorNext() { - this.latestStartExecuteTimeMs = nextExecuteTimeMs; - // todo The problem of delay should be considered. If it is greater than the ten-minute time window, - // should the task be lost or executed on a new time window? - this.nextExecuteTimeMs = latestStartExecuteTimeMs + intervalMs; - return nextExecuteTimeMs; - } - - public void pause() { - this.jobStatus = JobStatus.PAUSED; - } - - public void pause(String errMsg) { - this.jobStatus = JobStatus.PAUSED; - this.errMsg = errMsg; - } - - public void finish() { - this.jobStatus = JobStatus.FINISHED; - } - - public void resume() { - this.jobStatus = JobStatus.RUNNING; - } - - public void stop() { - this.jobStatus = JobStatus.STOPPED; - } - - public void checkJobParam() throws DdlException { - if (null == jobCategory) { - throw new DdlException("jobCategory must be set"); - } - if (null == executor) { - throw new DdlException("Job executor must be set"); - } - if (null == jobType) { - throw new DdlException("Job type must be set"); - } - if (jobType.equals(JobType.MANUAL)) { - return; - } - if (startTimeMs != 0L && startTimeMs < System.currentTimeMillis()) { - throw new DdlException("startTimeMs must be greater than current time"); - } - if (immediatelyStart && startTimeMs != 0L) { - throw new DdlException("immediately start and startTimeMs can't be set at the same time"); - } - if (immediatelyStart) { - startTimeMs = System.currentTimeMillis(); - } - if (endTimeMs != 0L && endTimeMs < System.currentTimeMillis()) { - throw new DdlException("endTimeMs must be greater than current time"); - } - if (null != intervalUnit && null != originInterval) { - this.intervalMs = intervalUnit.getParameterValue(originInterval); - } - if (jobType.equals(JobType.RECURRING) && (intervalMs == null || intervalMs <= 0L)) { - throw new DdlException("cycle job must set intervalMs"); - } - - } - - - @Override - public void write(DataOutput out) throws IOException { - String jobData = GsonUtils.GSON.toJson(this); - Text.writeString(out, jobData); - } - - public static Job readFields(DataInput in) throws IOException { - return GsonUtils.GSON.fromJson(Text.readString(in), Job.class); - } - - public List getShowInfo() { - List row = Lists.newArrayList(); - row.add(String.valueOf(jobId)); - row.add(jobName); - row.add(user); - row.add(jobType.name()); - - row.add(convertRecurringStrategyToString()); - row.add(jobStatus.name()); - row.add(null == lastExecuteTaskStatus ? "null" : lastExecuteTaskStatus.toString()); - row.add(createTimeMs <= 0L ? "null" : TimeUtils.longToTimeString(createTimeMs)); - row.add(comment == null ? "null" : comment); - return row; - } - - private String convertRecurringStrategyToString() { - if (jobType.equals(JobType.MANUAL)) { - return "MANUAL TRIGGER"; - } - switch (jobType) { - case ONE_TIME: - return "AT " + TimeUtils.longToTimeString(startTimeMs); - case RECURRING: - String result = "EVERY " + originInterval + " " + intervalUnit.name(); - if (startTimeMs > 0) { - result += " STARTS " + TimeUtils.longToTimeString(startTimeMs); - } - if (endTimeMs > 0) { - result += " ENDS " + TimeUtils.longToTimeString(endTimeMs); - } - return result; - case STREAMING: - return "STREAMING" + (startTimeMs > 0 ? " AT " + TimeUtils.longToTimeString(startTimeMs) : ""); - case MANUAL: - return "MANUAL TRIGGER"; - default: - return "UNKNOWN"; - } - } - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/job/JobTask.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/job/JobTask.java deleted file mode 100644 index 1f8aac5828..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/job/JobTask.java +++ /dev/null @@ -1,136 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.scheduler.job; - -import org.apache.doris.common.io.Text; -import org.apache.doris.common.io.Writable; -import org.apache.doris.common.util.TimeUtils; -import org.apache.doris.persist.gson.GsonUtils; -import org.apache.doris.scheduler.constants.TaskType; - -import com.google.common.collect.Lists; -import com.google.gson.annotations.SerializedName; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.List; - -@Data -@Slf4j -public class JobTask implements Writable { - - @SerializedName("jobId") - private Long jobId; - @SerializedName("taskId") - private Long taskId; - @SerializedName("createTimeMs") - private Long createTimeMs; - @SerializedName("startTimeMs") - private Long startTimeMs; - @SerializedName("endTimeMs") - private Long endTimeMs; - @SerializedName("successful") - private Boolean isSuccessful; - - @SerializedName("executeSql") - private String executeSql; - @SerializedName("executeResult") - private String executeResult; - @SerializedName("errorMsg") - private String errorMsg; - - @SerializedName("contextDataStr") - private String contextDataStr; - - @SerializedName("taskType") - private TaskType taskType = TaskType.SCHEDULER_JOB_TASK; - - /** - * Some parameters specific to the current task that need to be used to execute the task - * eg: sql task, sql it's: select * from table where id = 1 order by id desc limit ${limit} offset ${offset} - * contextData is a map, key1 is limit, value is 10,key2 is offset, value is 1 - * when execute the task, we will replace the ${limit} to 10, ${offset} to 1 - * so to execute sql is: select * from table where id = 1 order by id desc limit 10 offset 1. - */ - private T contextData; - - public JobTask(Long jobId, Long taskId, Long createTimeMs) { - //it's enough to use nanoTime to identify a task - this.taskId = taskId; - this.jobId = jobId; - this.createTimeMs = createTimeMs; - } - - public JobTask(Long jobId, Long taskId, Long createTimeMs, T contextData) { - this(jobId, taskId, createTimeMs); - this.contextData = contextData; - try { - this.contextDataStr = GsonUtils.GSON.toJson(contextData); - } catch (Exception e) { - this.contextDataStr = null; - log.error("contextData serialize failed, jobId: {}, taskId: {}", jobId, taskId, e); - } - } - - public List getShowInfo(String jobName) { - List row = Lists.newArrayList(); - row.add(String.valueOf(taskId)); - row.add(String.valueOf(jobId)); - row.add(jobName); - if (null != createTimeMs) { - row.add(TimeUtils.longToTimeString(createTimeMs)); - } - row.add(TimeUtils.longToTimeString(startTimeMs)); - row.add(null == endTimeMs ? "null" : TimeUtils.longToTimeString(endTimeMs)); - if (endTimeMs == null) { - row.add("RUNNING"); - } else { - row.add(isSuccessful ? "SUCCESS" : "FAILED"); - } - if (null == executeSql) { - row.add("null"); - } else { - row.add(executeSql); - } - if (null == executeResult) { - row.add("null"); - } else { - row.add(executeResult); - } - if (null == errorMsg) { - row.add("null"); - } else { - row.add(errorMsg); - } - row.add(taskType.name()); - return row; - } - - @Override - public void write(DataOutput out) throws IOException { - String jobData = GsonUtils.GSON.toJson(this); - Text.writeString(out, jobData); - } - - public static JobTask readFields(DataInput in) throws IOException { - return GsonUtils.GSON.fromJson(Text.readString(in), JobTask.class); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/job/TimerJobTask.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/job/TimerJobTask.java deleted file mode 100644 index b3a199a8a6..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/job/TimerJobTask.java +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.scheduler.job; - -import org.apache.doris.scheduler.disruptor.TaskDisruptor; - -import io.netty.util.Timeout; -import io.netty.util.TimerTask; -import lombok.Getter; - -/** - * This class represents a timer task that can be scheduled by a Netty timer. - * When the timer task is triggered, it produces a Job task using the Disruptor. - * The Job task contains the ID of the Job and the ID of the task itself. - */ -@Getter -public class TimerJobTask implements TimerTask { - - private final Long jobId; - - // more fields should be added here and record in feature - private final Long taskId; - - private final Long startTimestamp; - - private final TaskDisruptor taskDisruptor; - - public TimerJobTask(Long jobId, Long taskId, Long startTimestamp, TaskDisruptor taskDisruptor) { - this.jobId = jobId; - this.startTimestamp = startTimestamp; - this.taskDisruptor = taskDisruptor; - this.taskId = taskId; - } - - @Override - public void run(Timeout timeout) { - if (timeout.isCancelled()) { - return; - } - taskDisruptor.tryPublish(jobId, taskId); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/manager/JobTaskManager.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/manager/JobTaskManager.java deleted file mode 100644 index 6f7cd83907..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/manager/JobTaskManager.java +++ /dev/null @@ -1,152 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.scheduler.manager; - -import org.apache.doris.catalog.Env; -import org.apache.doris.common.Config; -import org.apache.doris.common.io.Writable; -import org.apache.doris.common.util.LogBuilder; -import org.apache.doris.common.util.LogKey; -import org.apache.doris.scheduler.job.JobTask; - -import lombok.extern.slf4j.Slf4j; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; - -@Slf4j -public class JobTaskManager implements Writable { - - private static final Integer TASK_MAX_NUM = Config.scheduler_job_task_max_saved_count; - - private ConcurrentHashMap> jobTaskMap = new ConcurrentHashMap<>(16); - - - /** - * taskId -> startTime - * used to record the start time of the task to be executed - * will clear when the task is executed - */ - private static ConcurrentHashMap> prepareTaskCreateMsMap = new ConcurrentHashMap<>(16); - - public static void addPrepareTask(JobTask jobTask) { - long jobId = jobTask.getJobId(); - long taskId = jobTask.getTaskId(); - prepareTaskCreateMsMap.computeIfAbsent(jobId, k -> new HashMap<>()); - prepareTaskCreateMsMap.get(jobId).put(taskId, jobTask); - } - - public static JobTask pollPrepareTaskByTaskId(Long jobId, Long taskId) { - if (!prepareTaskCreateMsMap.containsKey(jobId) || !prepareTaskCreateMsMap.get(jobId).containsKey(taskId)) { - // if the job is not in the map, return new JobTask - // return new JobTask(jobId, taskId, System.currentTimeMillis()); fixme - return null; - } - return prepareTaskCreateMsMap.get(jobId).remove(taskId); - } - - public static void clearPrepareTaskByJobId(Long jobId) { - prepareTaskCreateMsMap.remove(jobId); - } - - public void addJobTask(JobTask jobTask, boolean persist) { - ConcurrentLinkedQueue jobTasks = jobTaskMap - .computeIfAbsent(jobTask.getJobId(), k -> new ConcurrentLinkedQueue<>()); - jobTasks.add(jobTask); - if (jobTasks.size() > TASK_MAX_NUM) { - JobTask oldTask = jobTasks.poll(); - if (persist) { - Env.getCurrentEnv().getEditLog().logDeleteJobTask(oldTask); - } - } - if (persist) { - Env.getCurrentEnv().getEditLog().logCreateJobTask(jobTask); - } - } - - public List getJobTasks(Long jobId) { - if (jobTaskMap.containsKey(jobId)) { - ConcurrentLinkedQueue jobTasks = jobTaskMap.get(jobId); - List jobTaskList = new LinkedList<>(jobTasks); - Collections.reverse(jobTaskList); - return jobTaskList; - } - return new ArrayList<>(); - } - - public void replayCreateTask(JobTask task) { - ConcurrentLinkedQueue jobTasks = jobTaskMap - .computeIfAbsent(task.getJobId(), k -> new ConcurrentLinkedQueue<>()); - jobTasks.add(task); - log.info(new LogBuilder(LogKey.SCHEDULER_TASK, task.getTaskId()) - .add("msg", "replay create scheduler task").build()); - } - - public void replayDeleteTask(JobTask task) { - ConcurrentLinkedQueue jobTasks = jobTaskMap.get(task.getJobId()); - if (jobTasks != null) { - jobTasks.remove(task); - } - log.info(new LogBuilder(LogKey.SCHEDULER_TASK, task.getTaskId()) - .add("msg", "replay delete scheduler task").build()); - } - - public void deleteJobTasks(Long jobId) { - ConcurrentLinkedQueue jobTasks = jobTaskMap.get(jobId); - if (null != jobTasks) { - jobTaskMap.remove(jobId); - } - clearPrepareTaskByJobId(jobId); - } - - @Override - public void write(DataOutput out) throws IOException { - out.writeInt(jobTaskMap.size()); - for (Map.Entry> entry : jobTaskMap.entrySet()) { - out.writeLong(entry.getKey()); - out.writeInt(entry.getValue().size()); - for (JobTask jobTask : entry.getValue()) { - jobTask.write(out); - } - } - - } - - public void readFields(DataInput in) throws IOException { - int size = in.readInt(); - for (int i = 0; i < size; i++) { - Long jobId = in.readLong(); - int taskSize = in.readInt(); - ConcurrentLinkedQueue jobTasks = new ConcurrentLinkedQueue<>(); - for (int j = 0; j < taskSize; j++) { - JobTask jobTask = JobTask.readFields(in); - jobTasks.add(jobTask); - } - jobTaskMap.put(jobId, jobTasks); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/manager/TimerJobManager.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/manager/TimerJobManager.java deleted file mode 100644 index 33ac7d5f94..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/manager/TimerJobManager.java +++ /dev/null @@ -1,573 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.scheduler.manager; - -import org.apache.doris.catalog.Env; -import org.apache.doris.common.Config; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.PatternMatcher; -import org.apache.doris.common.io.Writable; -import org.apache.doris.common.util.LogBuilder; -import org.apache.doris.common.util.LogKey; -import org.apache.doris.common.util.TimeUtils; -import org.apache.doris.scheduler.constants.JobCategory; -import org.apache.doris.scheduler.constants.JobStatus; -import org.apache.doris.scheduler.constants.JobType; -import org.apache.doris.scheduler.constants.TaskType; -import org.apache.doris.scheduler.disruptor.TaskDisruptor; -import org.apache.doris.scheduler.job.Job; -import org.apache.doris.scheduler.job.JobTask; -import org.apache.doris.scheduler.job.TimerJobTask; - -import io.netty.util.HashedWheelTimer; -import io.netty.util.Timeout; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.io.Closeable; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; - -@Slf4j -public class TimerJobManager implements Closeable, Writable { - - private final ConcurrentHashMap jobMap = new ConcurrentHashMap<>(128); - private long lastBatchSchedulerTimestamp; - private static final long BATCH_SCHEDULER_INTERVAL_SECONDS = 600; - - /** - * batch scheduler interval ms time - */ - private static final long BATCH_SCHEDULER_INTERVAL_MILLI_SECONDS = BATCH_SCHEDULER_INTERVAL_SECONDS * 1000L; - - private boolean isClosed = false; - - /** - * key: jobid - * value: timeout list for one job - * it's used to cancel task, if task has started, it can't be canceled - */ - private final ConcurrentHashMap> jobTimeoutMap = new ConcurrentHashMap<>(128); - - /** - * scheduler tasks, it's used to scheduler job - */ - private HashedWheelTimer dorisTimer; - - /** - * Producer and Consumer model - * disruptor is used to handle task - * disruptor will start a thread pool to handle task - */ - @Setter - private TaskDisruptor disruptor; - - public TimerJobManager() { - this.lastBatchSchedulerTimestamp = System.currentTimeMillis(); - } - - public void start() { - dorisTimer = new HashedWheelTimer(1, TimeUnit.SECONDS, 660); - dorisTimer.start(); - Long currentTimeMs = System.currentTimeMillis(); - jobMap.forEach((jobId, job) -> { - Long nextExecuteTimeMs = findFistExecuteTime(currentTimeMs, job.getStartTimeMs(), - job.getIntervalMs(), job.getJobType()); - job.setNextExecuteTimeMs(nextExecuteTimeMs); - }); - batchSchedulerTasks(); - cycleSystemSchedulerTasks(); - } - - public Long registerJob(Job job) throws DdlException { - job.checkJobParam(); - checkIsJobNameUsed(job.getDbName(), job.getJobName(), job.getJobCategory()); - jobMap.putIfAbsent(job.getJobId(), job); - initAndSchedulerJob(job); - Env.getCurrentEnv().getEditLog().logCreateJob(job); - return job.getJobId(); - } - - public void replayCreateJob(Job job) { - if (jobMap.containsKey(job.getJobId())) { - return; - } - jobMap.putIfAbsent(job.getJobId(), job); - initAndSchedulerJob(job); - log.info(new LogBuilder(LogKey.SCHEDULER_JOB, job.getJobId()) - .add("msg", "replay create scheduler job").build()); - } - - /** - * Replay update load job. - **/ - public void replayUpdateJob(Job job) { - jobMap.put(job.getJobId(), job); - if (JobStatus.RUNNING.equals(job.getJobStatus())) { - initAndSchedulerJob(job); - } - log.info(new LogBuilder(LogKey.SCHEDULER_JOB, job.getJobId()) - .add("msg", "replay update scheduler job").build()); - } - - public void replayDeleteJob(Job job) { - if (null == jobMap.get(job.getJobId())) { - return; - } - jobMap.remove(job.getJobId()); - log.info(new LogBuilder(LogKey.SCHEDULER_JOB, job.getJobId()) - .add("msg", "replay delete scheduler job").build()); - Env.getCurrentEnv().getJobTaskManager().deleteJobTasks(job.getJobId()); - } - - private void checkIsJobNameUsed(String dbName, String jobName, JobCategory jobCategory) throws DdlException { - Optional optionalJob = jobMap.values().stream().filter(job -> job.getJobCategory().equals(jobCategory)) - .filter(job -> job.getDbName().equals(dbName)) - .filter(job -> job.getJobName().equals(jobName)).findFirst(); - if (optionalJob.isPresent()) { - throw new DdlException("Name " + jobName + " already used in db " + dbName); - } - } - - private void initAndSchedulerJob(Job job) { - if (!job.getJobStatus().equals(JobStatus.RUNNING) || job.getJobType().equals(JobType.MANUAL)) { - return; - } - - Long currentTimeMs = System.currentTimeMillis(); - Long nextExecuteTimeMs = findFistExecuteTime(currentTimeMs, job.getStartTimeMs(), - job.getIntervalMs(), job.getJobType()); - job.setNextExecuteTimeMs(nextExecuteTimeMs); - if (job.getNextExecuteTimeMs() < lastBatchSchedulerTimestamp) { - List executeTimestamp = findTasksBetweenTime(job, - lastBatchSchedulerTimestamp, - job.getNextExecuteTimeMs(), job.getJobType()); - if (!executeTimestamp.isEmpty()) { - for (Long timestamp : executeTimestamp) { - putOneTask(job.getJobId(), timestamp); - } - } - } - } - - private Long findFistExecuteTime(Long currentTimeMs, Long startTimeMs, Long intervalMs, JobType jobType) { - // if job not delay, first execute time is start time - if (startTimeMs != 0L && startTimeMs > currentTimeMs) { - return startTimeMs; - } - // if job already delay, first execute time is current time - if (startTimeMs != 0L && startTimeMs < currentTimeMs) { - return currentTimeMs; - } - // if it's cycle job and not set start tine, first execute time is current time + interval - if (jobType.equals(JobType.RECURRING) && startTimeMs == 0L) { - return currentTimeMs + intervalMs; - } - // if it's not cycle job and already delay, first execute time is current time - return currentTimeMs; - } - - public boolean immediateExecuteTask(Long jobId, T taskContextData) throws DdlException { - Job job = jobMap.get(jobId); - if (job == null) { - log.warn("immediateExecuteTask failed, jobId: {} not exist", jobId); - return false; - } - if (!job.getJobStatus().equals(JobStatus.RUNNING)) { - log.warn("immediateExecuteTask failed, jobId: {} is not running", jobId); - return false; - } - JobTask jobTask = createInitialTask(jobId, taskContextData); - jobTask.setTaskType(TaskType.MANUAL_JOB_TASK); - JobTaskManager.addPrepareTask(jobTask); - disruptor.tryPublish(jobId, jobTask.getTaskId(), TaskType.MANUAL_JOB_TASK); - return true; - } - - public void unregisterJob(Long jobId) { - jobMap.remove(jobId); - } - - public void pauseJob(Long jobId) { - Job job = jobMap.get(jobId); - if (jobMap.get(jobId) == null) { - log.warn("pauseJob failed, jobId: {} not exist", jobId); - } - if (jobMap.get(jobId).getJobStatus().equals(JobStatus.PAUSED)) { - log.warn("pauseJob failed, jobId: {} is already paused", jobId); - } - pauseJob(job); - } - - public void setJobLatestStatus(long jobId, boolean status) { - Job job = jobMap.get(jobId); - if (jobMap.get(jobId) == null) { - log.warn("pauseJob failed, jobId: {} not exist", jobId); - } - job.setLastExecuteTaskStatus(status); - } - - public void stopJob(String dbName, String jobName, JobCategory jobCategory) throws DdlException { - Optional optionalJob = findJob(dbName, jobName, jobCategory); - - if (!optionalJob.isPresent()) { - throw new DdlException("Job " + jobName + " not exist in db " + dbName); - } - Job job = optionalJob.get(); - if (job.getJobStatus().equals(JobStatus.STOPPED)) { - throw new DdlException("Job " + jobName + " is already stopped"); - } - stopJob(optionalJob.get()); - Env.getCurrentEnv().getEditLog().logDeleteJob(optionalJob.get()); - } - - private void stopJob(Job job) { - if (JobStatus.RUNNING.equals(job.getJobStatus())) { - cancelJobAllTask(job.getJobId()); - } - job.setJobStatus(JobStatus.STOPPED); - jobMap.get(job.getJobId()).stop(); - Env.getCurrentEnv().getEditLog().logDeleteJob(job); - Env.getCurrentEnv().getJobTaskManager().deleteJobTasks(job.getJobId()); - } - - - public void resumeJob(String dbName, String jobName, JobCategory jobCategory) throws DdlException { - Optional optionalJob = findJob(dbName, jobName, jobCategory); - if (!optionalJob.isPresent()) { - throw new DdlException("Job " + jobName + " not exist in db " + dbName); - } - Job job = optionalJob.get(); - if (!job.getJobStatus().equals(JobStatus.PAUSED)) { - throw new DdlException("Job " + jobName + " is not paused"); - } - resumeJob(job); - } - - private void resumeJob(Job job) { - cancelJobAllTask(job.getJobId()); - job.setJobStatus(JobStatus.RUNNING); - jobMap.get(job.getJobId()).resume(); - initAndSchedulerJob(job); - Env.getCurrentEnv().getEditLog().logUpdateJob(job); - } - - public void pauseJob(String dbName, String jobName, JobCategory jobCategory) throws DdlException { - Optional optionalJob = findJob(dbName, jobName, jobCategory); - if (!optionalJob.isPresent()) { - throw new DdlException("Job " + jobName + " not exist in db " + dbName); - } - Job job = optionalJob.get(); - if (!job.getJobStatus().equals(JobStatus.RUNNING)) { - throw new DdlException("Job " + jobName + " is not running"); - } - pauseJob(job); - } - - private void pauseJob(Job job) { - cancelJobAllTask(job.getJobId()); - job.setJobStatus(JobStatus.PAUSED); - jobMap.get(job.getJobId()).pause(); - Env.getCurrentEnv().getEditLog().logUpdateJob(job); - } - - public void finishJob(long jobId) { - Job job = jobMap.get(jobId); - if (jobMap.get(jobId) == null) { - log.warn("update job status failed, jobId: {} not exist", jobId); - } - if (jobMap.get(jobId).getJobStatus().equals(JobStatus.FINISHED)) { - return; - } - job.setLatestCompleteExecuteTimeMs(System.currentTimeMillis()); - cancelJobAllTask(job.getJobId()); - job.setJobStatus(JobStatus.FINISHED); - Env.getCurrentEnv().getEditLog().logUpdateJob(job); - } - - private Optional findJob(String dbName, String jobName, JobCategory jobCategory) { - return jobMap.values().stream().filter(job -> checkJobMatch(job, dbName, jobName, jobCategory)).findFirst(); - } - - private boolean checkJobMatch(Job job, String dbName, String jobName, JobCategory jobCategory) { - return job.getDbName().equals(dbName) && job.getJobName().equals(jobName) - && job.getJobCategory().equals(jobCategory); - } - - - public void resumeJob(Long jobId) { - if (jobMap.get(jobId) == null) { - log.warn("resumeJob failed, jobId: {} not exist", jobId); - return; - } - Job job = jobMap.get(jobId); - resumeJob(job); - } - - public void stopJob(Long jobId) { - Job job = jobMap.get(jobId); - if (null == job) { - log.warn("stopJob failed, jobId: {} not exist", jobId); - return; - } - if (job.getJobStatus().equals(JobStatus.STOPPED)) { - log.warn("stopJob failed, jobId: {} is already stopped", jobId); - return; - } - stopJob(job); - } - - public Job getJob(Long jobId) { - return jobMap.get(jobId); - } - - public Map getAllJob() { - return jobMap; - } - - public void batchSchedulerTasks() { - executeJobIdsWithinLastTenMinutesWindow(); - } - - private List findTasksBetweenTime(Job job, Long endTimeEndWindow, Long nextExecuteTime, JobType jobType) { - - List jobExecuteTimes = new ArrayList<>(); - if (!jobType.equals(JobType.RECURRING) && (nextExecuteTime < endTimeEndWindow)) { - jobExecuteTimes.add(nextExecuteTime); - return jobExecuteTimes; - } - if (jobType.equals(JobType.RECURRING) && (nextExecuteTime > endTimeEndWindow)) { - return new ArrayList<>(); - } - while (endTimeEndWindow >= nextExecuteTime) { - if (job.isTaskTimeExceeded()) { - break; - } - jobExecuteTimes.add(nextExecuteTime); - nextExecuteTime = job.getExecuteTimestampAndGeneratorNext(); - } - return jobExecuteTimes; - } - - /** - * We will get the task in the next time window, and then hand it over to the time wheel for timing trigger - */ - private void executeJobIdsWithinLastTenMinutesWindow() { - // if the task executes for more than 10 minutes, it will be delay, so, - // set lastBatchSchedulerTimestamp to current time - if (lastBatchSchedulerTimestamp < System.currentTimeMillis()) { - this.lastBatchSchedulerTimestamp = System.currentTimeMillis(); - } - this.lastBatchSchedulerTimestamp += BATCH_SCHEDULER_INTERVAL_MILLI_SECONDS; - if (jobMap.isEmpty()) { - return; - } - jobMap.forEach((k, v) -> { - if (!v.getJobType().equals(JobType.MANUAL) && v.isRunning() && (v.getNextExecuteTimeMs() - + v.getIntervalMs() < lastBatchSchedulerTimestamp)) { - List executeTimes = findTasksBetweenTime( - v, lastBatchSchedulerTimestamp, - v.getNextExecuteTimeMs(), v.getJobType()); - if (!executeTimes.isEmpty()) { - for (Long executeTime : executeTimes) { - putOneTask(v.getJobId(), executeTime); - } - } - } - }); - } - - /** - * We will cycle system scheduler tasks every 10 minutes. - * Jobs will be re-registered after the task is completed - */ - private void cycleSystemSchedulerTasks() { - log.info("re-register system scheduler tasks" + TimeUtils.longToTimeString(System.currentTimeMillis())); - dorisTimer.newTimeout(timeout -> { - batchSchedulerTasks(); - clearFinishJob(); - cycleSystemSchedulerTasks(); - }, BATCH_SCHEDULER_INTERVAL_SECONDS, TimeUnit.SECONDS); - - } - - /** - * put one task to time wheel,it's well be trigger after delay milliseconds - * if the scheduler is closed, the task will not be put into the time wheel - * if delay is less than 0, the task will be trigger immediately - * - * @param jobId job id, we will use it to find the job - * @param startExecuteTime the task will be trigger in this time, unit is millisecond,and we will convert it to - * delay seconds, we just can be second precision - */ - public void putOneTask(Long jobId, Long startExecuteTime) { - if (isClosed) { - log.info("putOneTask failed, scheduler is closed, jobId: {}", jobId); - return; - } - JobTask jobTask = createAsyncInitialTask(jobId, startExecuteTime); - long taskId = jobTask.getTaskId(); - TimerJobTask task = new TimerJobTask(jobId, taskId, startExecuteTime, disruptor); - long delay = getDelaySecond(task.getStartTimestamp()); - Timeout timeout = dorisTimer.newTimeout(task, delay, TimeUnit.SECONDS); - if (timeout == null) { - log.error("putOneTask failed, jobId: {}", task.getJobId()); - return; - } - if (jobTimeoutMap.containsKey(task.getJobId())) { - jobTimeoutMap.get(task.getJobId()).put(task.getTaskId(), timeout); - JobTaskManager.addPrepareTask(jobTask); - return; - } - Map timeoutMap = new ConcurrentHashMap<>(); - timeoutMap.put(task.getTaskId(), timeout); - jobTimeoutMap.put(task.getJobId(), timeoutMap); - JobTaskManager.addPrepareTask(jobTask); - } - - // cancel all task for one job - // if task has started, it can't be canceled - public void cancelJobAllTask(Long jobId) { - if (!jobTimeoutMap.containsKey(jobId)) { - return; - } - - jobTimeoutMap.get(jobId).values().forEach(timeout -> { - if (!timeout.isExpired() || timeout.isCancelled()) { - timeout.cancel(); - } - }); - JobTaskManager.clearPrepareTaskByJobId(jobId); - } - - // get delay time, if startTimestamp is less than now, return 0 - private long getDelaySecond(long startTimestamp) { - long delay = 0; - long now = System.currentTimeMillis(); - if (startTimestamp > now) { - delay = startTimestamp - now; - } else { - //if execute time is less than now, return 0,immediately execute - log.warn("startTimestamp is less than now, startTimestamp: {}, now: {}", startTimestamp, now); - return delay; - } - return delay / 1000; - } - - @Override - public void close() throws IOException { - isClosed = true; - dorisTimer.stop(); - disruptor.close(); - } - - /** - * sort by job id - * - * @param dbFullName database name - * @param category job category - * @param matcher job name matcher - */ - public List queryJob(String dbFullName, String jobName, JobCategory category, PatternMatcher matcher) { - List jobs = new ArrayList<>(); - jobMap.values().forEach(job -> { - if (matchJob(job, dbFullName, jobName, category, matcher)) { - jobs.add(job); - } - }); - return jobs; - } - - private boolean matchJob(Job job, String dbFullName, String jobName, JobCategory category, PatternMatcher matcher) { - if (StringUtils.isNotBlank(dbFullName) && !job.getDbName().equalsIgnoreCase(dbFullName)) { - return false; - } - if (StringUtils.isNotBlank(jobName) && !job.getJobName().equalsIgnoreCase(jobName)) { - return false; - } - if (category != null && !job.getJobCategory().equals(category)) { - return false; - } - return null == matcher || matcher.match(job.getJobName()); - } - - public void putOneJobToQueen(Long jobId) { - JobTask jobTask = createInitialTask(jobId, null); - JobTaskManager.addPrepareTask(jobTask); - disruptor.tryPublish(jobId, jobTask.getTaskId()); - } - - private JobTask createAsyncInitialTask(long jobId, long createTimeMs) { - long taskId = System.nanoTime(); - return new JobTask(jobId, taskId, createTimeMs); - } - - private JobTask createInitialTask(long jobId, T context) { - long taskId = System.nanoTime(); - return new JobTask(jobId, taskId, System.currentTimeMillis(), context); - } - - @Override - public void write(DataOutput out) throws IOException { - out.writeInt(jobMap.size()); - for (Job job : jobMap.values()) { - job.write(out); - } - } - - /** - * read job from data input, and init job - * - * @param in data input - * @throws IOException io exception when read data input error - */ - public void readFields(DataInput in) throws IOException { - int size = in.readInt(); - for (int i = 0; i < size; i++) { - Job job = Job.readFields(in); - jobMap.putIfAbsent(job.getJobId(), job); - } - } - - /** - * clear finish job，if job finish time is more than @Config.finish_job_max_saved_second, we will delete it - * this method will be called every 10 minutes, therefore, the actual maximum - * deletion time is Config.finish_job_max_saved_second + 10 min. - * we could to delete job in time, but it's not make sense.start - */ - private void clearFinishJob() { - Long now = System.currentTimeMillis(); - jobMap.values().forEach(job -> { - if (job.isFinished() && now - job.getLatestCompleteExecuteTimeMs() > Config.finish_job_max_saved_second) { - jobMap.remove(job.getJobId()); - Env.getCurrentEnv().getEditLog().logDeleteJob(job); - Env.getCurrentEnv().getJobTaskManager().deleteJobTasks(job.getJobId()); - log.debug("delete finish job:{}", job.getJobId()); - } - }); - - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/manager/TransientTaskManager.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/manager/TransientTaskManager.java index bc72689622..9602b19ca2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/manager/TransientTaskManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/scheduler/manager/TransientTaskManager.java @@ -43,6 +43,8 @@ public class TransientTaskManager { private TaskDisruptor disruptor; public TransientTaskManager() { + disruptor = new TaskDisruptor(this); + disruptor.start(); } public TransientTaskExecutor getMemoryTaskExecutor(Long taskId) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/registry/PersistentJobRegister.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/registry/PersistentJobRegister.java deleted file mode 100644 index f1a901299f..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/registry/PersistentJobRegister.java +++ /dev/null @@ -1,136 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.scheduler.registry; - -import org.apache.doris.common.DdlException; -import org.apache.doris.common.PatternMatcher; -import org.apache.doris.scheduler.constants.JobCategory; -import org.apache.doris.scheduler.executor.JobExecutor; -import org.apache.doris.scheduler.job.Job; - -import java.io.IOException; -import java.util.List; - -/** - * This interface provides a contract for registering timed scheduling events. - * The implementation should trigger events in a timely manner using a specific algorithm. - * The execution of the events may be asynchronous and not guarantee strict timing accuracy. - */ -public interface PersistentJobRegister { - - /** - * Register a job - * - * @param name job name,it's not unique - * @param intervalMs job interval, unit: ms - * @param executor job executor @See {@link JobExecutor} - * @return event job id - */ - Long registerJob(String name, Long intervalMs, JobExecutor executor) throws DdlException; - - /** - * Register a job - * - * @param name job name,it's not unique - * @param intervalMs job interval, unit: ms - * @param startTimeStamp job start time stamp, unit: ms - * if startTimeStamp is null, event job will start immediately in the next cycle - * startTimeStamp should be greater than current time - * @param executor event job executor @See {@link JobExecutor} - * @return job id - */ - Long registerJob(String name, Long intervalMs, Long startTimeStamp, JobExecutor executor) throws DdlException; - - - /** - * Register a event job - * - * @param name job name,it's not unique - * @param intervalMs job interval, unit: ms - * @param startTimeStamp job start time stamp, unit: ms - * if startTimeStamp is null, job will start immediately in the next cycle - * startTimeStamp should be greater than current time - * @param endTimeStamp job end time stamp, unit: ms - * if endTimeStamp is null, job will never stop - * endTimeStamp must be greater than startTimeStamp and endTimeStamp should be greater - * than current time - * @param executor event job executor @See {@link JobExecutor} - * @return event job id - */ - Long registerJob(String name, Long intervalMs, Long startTimeStamp, Long endTimeStamp, - JobExecutor executor) throws DdlException; - - /** - * if job is running, pause it - * pause means event job will not be executed in the next cycle,but current cycle will not be interrupted - * we can resume it by {@link #resumeJob(Long)} - * - * @param jodId job id - * if jobId not exist, return false - */ - void pauseJob(Long jodId); - - void pauseJob(String dbName, String jobName, JobCategory jobCategory) throws DdlException; - - void resumeJob(String dbName, String jobName, JobCategory jobCategory) throws DdlException; - - /** - * if job is running, stop it - * stop means event job will not be executed in the next cycle and current cycle will be interrupted - * stop not can be resumed, if you want to resume it, you should register it again - * we will delete stopped event job - * - * @param jobId event job id - */ - void stopJob(Long jobId); - - void stopJob(String dbName, String jobName, JobCategory jobCategory) throws DdlException; - - /** - * if job is paused, resume it - * - * @param jobId job id - */ - void resumeJob(Long jobId); - - Long registerJob(Job job) throws DdlException; - - - /** - * execute job task immediately,this method will not change job status and don't affect scheduler job - * this task type should set to {@link org.apache.doris.scheduler.constants.TaskType#MANUAL_JOB_TASK} - * - * @param jobId job id - * @param contextData if you need to pass parameters to the task, - * @param context data type - * @return true if execute success, false if execute failed, - * if job is not exist or job is not running, or job not support manual execute, return false - */ - boolean immediateExecuteTask(Long jobId, T contextData) throws DdlException; - - List getJobs(String dbFullName, String jobName, JobCategory jobCategory, PatternMatcher matcher); - - /** - * close job scheduler register - * close means job scheduler register will not accept new job - * Jobs that have not reached the trigger time will not be executed. Jobs that have reached the trigger time will - * have an execution time of 5 seconds, and will not be executed if the time exceeds - */ - void close() throws IOException; - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/registry/TimerJobRegister.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/registry/TimerJobRegister.java deleted file mode 100644 index f8ab59e5d5..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/registry/TimerJobRegister.java +++ /dev/null @@ -1,115 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.scheduler.registry; - -import org.apache.doris.common.DdlException; -import org.apache.doris.common.PatternMatcher; -import org.apache.doris.scheduler.constants.JobCategory; -import org.apache.doris.scheduler.executor.JobExecutor; -import org.apache.doris.scheduler.job.Job; -import org.apache.doris.scheduler.manager.TimerJobManager; - -import lombok.extern.slf4j.Slf4j; - -import java.io.IOException; -import java.util.List; - -/** - * This class registers timed scheduling events using the Netty time wheel algorithm to trigger events in a timely - * manner. - * After the event is triggered, it is produced by the Disruptor producer and consumed by the consumer, which is an - * asynchronous - * consumption model that does not guarantee strict timing accuracy. - */ -@Slf4j -public class TimerJobRegister implements PersistentJobRegister { - - private final TimerJobManager timerJobManager; - - public TimerJobRegister(TimerJobManager timerJobManager) { - this.timerJobManager = timerJobManager; - } - - @Override - public Long registerJob(String name, Long intervalMs, JobExecutor executor) throws DdlException { - return this.registerJob(name, intervalMs, null, null, executor); - } - - @Override - public Long registerJob(String name, Long intervalMs, Long startTimeMs, JobExecutor executor) throws DdlException { - return this.registerJob(name, intervalMs, startTimeMs, null, executor); - } - - @Override - public Long registerJob(String name, Long intervalMs, Long startTimeMs, Long endTimeStamp, - JobExecutor executor) throws DdlException { - - Job job = new Job(name, intervalMs, startTimeMs, endTimeStamp, executor); - return timerJobManager.registerJob(job); - } - - @Override - public Long registerJob(Job job) throws DdlException { - return timerJobManager.registerJob(job); - } - - @Override - public boolean immediateExecuteTask(Long jobId, T data) throws DdlException { - return timerJobManager.immediateExecuteTask(jobId, data); - } - - @Override - public void pauseJob(Long jobId) { - timerJobManager.pauseJob(jobId); - } - - @Override - public void pauseJob(String dbName, String jobName, JobCategory jobCategory) throws DdlException { - timerJobManager.pauseJob(dbName, jobName, jobCategory); - } - - @Override - public void resumeJob(String dbName, String jobName, JobCategory jobCategory) throws DdlException { - timerJobManager.resumeJob(dbName, jobName, jobCategory); - } - - @Override - public void stopJob(Long jobId) { - timerJobManager.stopJob(jobId); - } - - @Override - public void stopJob(String dbName, String jobName, JobCategory jobCategory) throws DdlException { - timerJobManager.stopJob(dbName, jobName, jobCategory); - } - - @Override - public void resumeJob(Long jobId) { - timerJobManager.resumeJob(jobId); - } - - @Override - public List getJobs(String dbFullName, String jobName, JobCategory jobCategory, PatternMatcher matcher) { - return timerJobManager.queryJob(dbFullName, jobName, jobCategory, matcher); - } - - @Override - public void close() throws IOException { - timerJobManager.close(); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateJobStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateJobStmtTest.java index deca01acf4..f0e800cd6f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateJobStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateJobStmtTest.java @@ -31,8 +31,8 @@ public class CreateJobStmtTest { public void createOnceTimeJobStmt() throws Exception { String sql = "CREATE JOB job1 ON SCHEDULER AT \"2023-02-15\" DO SELECT * FROM `address` ;"; CreateJobStmt jobStmt = sqlParse(sql); - System.out.println(jobStmt.getStmt().toSql()); - Assertions.assertEquals("SELECT * FROM `address`", jobStmt.getStmt().toSql()); + System.out.println(jobStmt.getDoStmt().toSql()); + Assertions.assertEquals("SELECT * FROM `address`", jobStmt.getDoStmt().toSql()); String badExecuteSql = "CREATE JOB job1 ON SCHEDULER AT \"2023-02-15\" DO selects * from address ;"; Assertions.assertThrows(AnalysisException.class, () -> { @@ -56,16 +56,16 @@ public class CreateJobStmtTest { public void createCycleJob() throws Exception { String sql = "CREATE JOB job1 ON SCHEDULER EVERY 1 SECOND STARTS \"2023-02-15\" DO SELECT * FROM `address` ;"; CreateJobStmt jobStmt = sqlParse(sql); - Assertions.assertEquals("SELECT * FROM `address`", jobStmt.getStmt().toSql()); + Assertions.assertEquals("SELECT * FROM `address`", jobStmt.getDoStmt().toSql()); sql = "CREATE JOB job1 ON SCHEDULER EVERY 1 SECOND ENDS \"2023-02-15\" DO SELECT * FROM `address` ;"; jobStmt = sqlParse(sql); - Assertions.assertEquals("SELECT * FROM `address`", jobStmt.getStmt().toSql()); + Assertions.assertEquals("SELECT * FROM `address`", jobStmt.getDoStmt().toSql()); sql = "CREATE JOB job1 ON SCHEDULER EVERY 1 SECOND STARTS \"2023-02-15\" ENDS \"2023-02-16\" DO SELECT * FROM `address` ;"; jobStmt = sqlParse(sql); - Assertions.assertEquals("SELECT * FROM `address`", jobStmt.getStmt().toSql()); + Assertions.assertEquals("SELECT * FROM `address`", jobStmt.getDoStmt().toSql()); sql = "CREATE JOB job1 ON SCHEDULER EVERY 1 SECOND DO SELECT * FROM `address` ;"; jobStmt = sqlParse(sql); - Assertions.assertEquals("SELECT * FROM `address`", jobStmt.getStmt().toSql()); + Assertions.assertEquals("SELECT * FROM `address`", jobStmt.getDoStmt().toSql()); String badExecuteSql = "CREATE JOB job1 ON SCHEDULER AT \"2023-02-15\" DO selects * from address ;"; Assertions.assertThrows(AnalysisException.class, () -> { sqlParse(badExecuteSql); diff --git a/fe/fe-core/src/test/java/org/apache/doris/job/base/JobExecutionConfigurationTest.java b/fe/fe-core/src/test/java/org/apache/doris/job/base/JobExecutionConfigurationTest.java new file mode 100644 index 0000000000..eadc6c567d --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/job/base/JobExecutionConfigurationTest.java @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.job.base; + +import org.apache.doris.job.common.IntervalUnit; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +public class JobExecutionConfigurationTest { + + @Test + public void testGetTriggerDelayTimesOneTime() { + JobExecutionConfiguration configuration = new JobExecutionConfiguration(); + configuration.setExecuteType(JobExecuteType.ONE_TIME); + + TimerDefinition timerDefinition = new TimerDefinition(); + timerDefinition.setStartTimeMs(System.currentTimeMillis() + 1000); // Start time set to 1 second in the future + configuration.setTimerDefinition(timerDefinition); + + List delayTimes = configuration.getTriggerDelayTimes( + System.currentTimeMillis(), System.currentTimeMillis(), System.currentTimeMillis() + 5000); + + Assertions.assertEquals(1, delayTimes.size()); + Assertions.assertEquals(1, delayTimes.get(0).longValue()); + } + + @Test + public void testGetTriggerDelayTimesRecurring() { + JobExecutionConfiguration configuration = new JobExecutionConfiguration(); + configuration.setExecuteType(JobExecuteType.RECURRING); + + TimerDefinition timerDefinition = new TimerDefinition(); + timerDefinition.setStartTimeMs(1000L); // Start time set to 1 second in the future + timerDefinition.setInterval(10L); // Interval set to 10 milliseconds + timerDefinition.setIntervalUnit(IntervalUnit.SECOND); + configuration.setTimerDefinition(timerDefinition); + + List delayTimes = configuration.getTriggerDelayTimes( + 0L, 0L, 11000L); + + Assertions.assertEquals(2, delayTimes.size()); + Assertions.assertArrayEquals(new Long[]{1L, 11L}, delayTimes.toArray()); + delayTimes = configuration.getTriggerDelayTimes( + 2000L, 0L, 11000L); + Assertions.assertEquals(1, delayTimes.size()); + Assertions.assertArrayEquals(new Long[]{ 9L}, delayTimes.toArray()); + delayTimes = configuration.getTriggerDelayTimes( + 1001L, 0L, 10000L); + Assertions.assertEquals(0, delayTimes.size()); + } + +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/scheduler/disruptor/JobTest.java b/fe/fe-core/src/test/java/org/apache/doris/scheduler/disruptor/JobTest.java deleted file mode 100644 index 57fbfda6cf..0000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/scheduler/disruptor/JobTest.java +++ /dev/null @@ -1,77 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.scheduler.disruptor; - -import org.apache.doris.scheduler.common.IntervalUnit; -import org.apache.doris.scheduler.constants.JobCategory; -import org.apache.doris.scheduler.constants.JobType; -import org.apache.doris.scheduler.executor.SqlJobExecutor; -import org.apache.doris.scheduler.job.Job; - -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; - -public class JobTest { - - private static Job job; - - @BeforeAll - public static void init() { - SqlJobExecutor sqlJobExecutor = new SqlJobExecutor("insert into test values(1);"); - job = new Job("insertTest", 1000L, System.currentTimeMillis(), System.currentTimeMillis() + 100000, sqlJobExecutor); - job.setJobType(JobType.RECURRING); - job.setComment("test"); - job.setOriginInterval(10L); - job.setIntervalUnit(IntervalUnit.SECOND); - job.setUser("root"); - job.setDbName("test"); - job.setTimezone("Asia/Shanghai"); - job.setJobCategory(JobCategory.SQL); - } - - @Test - public void testSerialization() throws IOException { - Path path = Paths.get("./scheduler-jobs"); - Files.deleteIfExists(path); - Files.createFile(path); - DataOutputStream dos = new DataOutputStream(Files.newOutputStream(path)); - job.write(dos); - dos.flush(); - dos.close(); - DataInputStream dis = new DataInputStream(Files.newInputStream(path)); - Job readJob = Job.readFields(dis); - Assertions.assertEquals(job.getJobName(), readJob.getJobName()); - Assertions.assertEquals(job.getTimezone(), readJob.getTimezone()); - - } - - @AfterAll - public static void clean() throws IOException { - Path path = Paths.get("./scheduler-jobs"); - Files.deleteIfExists(path); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/scheduler/disruptor/TaskDisruptorTest.java b/fe/fe-core/src/test/java/org/apache/doris/scheduler/disruptor/TaskDisruptorTest.java deleted file mode 100644 index b8482b11ff..0000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/scheduler/disruptor/TaskDisruptorTest.java +++ /dev/null @@ -1,95 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.scheduler.disruptor; - -import org.apache.doris.catalog.Env; -import org.apache.doris.scheduler.constants.JobCategory; -import org.apache.doris.scheduler.exception.JobException; -import org.apache.doris.scheduler.executor.JobExecutor; -import org.apache.doris.scheduler.job.ExecutorResult; -import org.apache.doris.scheduler.job.Job; -import org.apache.doris.scheduler.job.JobTask; -import org.apache.doris.scheduler.manager.JobTaskManager; -import org.apache.doris.scheduler.manager.TimerJobManager; -import org.apache.doris.scheduler.manager.TransientTaskManager; - -import mockit.Expectations; -import mockit.Injectable; -import mockit.Mocked; -import mockit.Tested; -import org.awaitility.Awaitility; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.util.concurrent.TimeUnit; - -public class TaskDisruptorTest { - - @Tested - private TaskDisruptor taskDisruptor; - - @Injectable - private TimerJobManager timerJobManager; - - @Injectable - private TransientTaskManager transientTaskManager; - - private static boolean testEventExecuteFlag = false; - - @Mocked - Env env; - - @BeforeEach - public void init() { - taskDisruptor = new TaskDisruptor(timerJobManager, transientTaskManager); - taskDisruptor.start(); - } - - @Test - void testPublishEventAndConsumer() { - Job job = new Job("test", 6000L, null, - null, new TestExecutor()); - JobTask jobTask = new JobTask(job.getJobId(), 1L, System.currentTimeMillis()); - JobTaskManager.addPrepareTask(jobTask); - job.setJobCategory(JobCategory.COMMON); - new Expectations() {{ - timerJobManager.getJob(anyLong); - result = job; - }}; - taskDisruptor.tryPublish(job.getJobId(), 1L); - Awaitility.await().atMost(3, TimeUnit.SECONDS).until(() -> testEventExecuteFlag); - Assertions.assertTrue(testEventExecuteFlag); - } - - - class TestExecutor implements JobExecutor { - - @Override - public ExecutorResult execute(Job job, String dataContext) throws JobException { - testEventExecuteFlag = true; - return new ExecutorResult(true, true, null, "null"); - } - } - - @AfterEach - public void after() { - taskDisruptor.close(); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/scheduler/disruptor/TimerJobManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/scheduler/disruptor/TimerJobManagerTest.java deleted file mode 100644 index 5fcf242fd1..0000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/scheduler/disruptor/TimerJobManagerTest.java +++ /dev/null @@ -1,182 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.scheduler.disruptor; - -import org.apache.doris.catalog.Env; -import org.apache.doris.common.DdlException; -import org.apache.doris.persist.EditLog; -import org.apache.doris.scheduler.constants.JobCategory; -import org.apache.doris.scheduler.constants.JobType; -import org.apache.doris.scheduler.exception.JobException; -import org.apache.doris.scheduler.executor.JobExecutor; -import org.apache.doris.scheduler.job.ExecutorResult; -import org.apache.doris.scheduler.job.Job; -import org.apache.doris.scheduler.job.JobTask; -import org.apache.doris.scheduler.manager.TimerJobManager; -import org.apache.doris.scheduler.manager.TransientTaskManager; - -import lombok.extern.slf4j.Slf4j; -import mockit.Expectations; -import mockit.Mocked; -import org.awaitility.Awaitility; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -@Slf4j -public class TimerJobManagerTest { - - TimerJobManager timerJobManager; - - @Mocked - EditLog editLog; - - private static AtomicInteger testExecuteCount = new AtomicInteger(0); - Job job = new Job("test", 6000L, null, - null, new TestExecutor()); - JobTask jobTask = new JobTask(job.getJobId(), 1L, System.currentTimeMillis()); - - @BeforeEach - public void init() { - job.setJobType(JobType.RECURRING); - job.setJobCategory(JobCategory.COMMON); - testExecuteCount.set(0); - timerJobManager = new TimerJobManager(); - TransientTaskManager transientTaskManager = new TransientTaskManager(); - TaskDisruptor taskDisruptor = new TaskDisruptor(this.timerJobManager, transientTaskManager); - this.timerJobManager.setDisruptor(taskDisruptor); - taskDisruptor.start(); - timerJobManager.start(); - } - - @Test - public void testCycleScheduler(@Mocked Env env) throws DdlException { - setContext(env); - timerJobManager.registerJob(job); - //consider the time of the first execution and give some buffer time - Awaitility.await().atMost(30, TimeUnit.SECONDS).until(() -> testExecuteCount.get() >= 3); - } - - private void setContext(Env env) { - new Expectations() { - { - env.getEditLog(); - result = editLog; - editLog.logCreateJob((Job) any); - } - }; - } - - @Test - public void testCycleSchedulerAndStop(@Mocked Env env) throws DdlException { - setContext(env); - timerJobManager.registerJob(job); - long startTime = System.currentTimeMillis(); - Awaitility.await().atMost(8, TimeUnit.SECONDS).until(() -> testExecuteCount.get() >= 1); - timerJobManager.unregisterJob(job.getJobId()); - //consider the time of the first execution and give some buffer time - Awaitility.await().atMost(30, TimeUnit.SECONDS).until(() -> System.currentTimeMillis() >= startTime + 13000L); - Assertions.assertEquals(1, testExecuteCount.get()); - } - - @Test - public void testCycleSchedulerWithIncludeStartTimeAndEndTime(@Mocked Env env) throws DdlException { - setContext(env); - job.setStartTimeMs(System.currentTimeMillis() + 6000L); - long endTimestamp = System.currentTimeMillis() + 19000L; - job.setEndTimeMs(endTimestamp); - timerJobManager.registerJob(job); - //consider the time of the first execution and give some buffer time - - Awaitility.await().atMost(60, TimeUnit.SECONDS).until(() -> System.currentTimeMillis() - >= endTimestamp + 12000L); - Assertions.assertEquals(3, testExecuteCount.get()); - } - - @Test - public void testCycleSchedulerWithIncludeEndTime(@Mocked Env env) throws DdlException { - setContext(env); - long endTimestamp = System.currentTimeMillis() + 13000; - job.setEndTimeMs(endTimestamp); - timerJobManager.registerJob(job); - - //consider the time of the first execution and give some buffer time - Awaitility.await().atMost(36, TimeUnit.SECONDS).until(() -> System.currentTimeMillis() - >= endTimestamp + 12000L); - Assertions.assertEquals(2, testExecuteCount.get()); - } - - @Test - public void testCycleSchedulerWithIncludeStartTime(@Mocked Env env) throws DdlException { - setContext(env); - - long startTimestamp = System.currentTimeMillis() + 6000L; - job.setStartTimeMs(startTimestamp); - timerJobManager.registerJob(job); - //consider the time of the first execution and give some buffer time - Awaitility.await().atMost(14, TimeUnit.SECONDS).until(() -> System.currentTimeMillis() - >= startTimestamp + 7000L); - Assertions.assertEquals(2, testExecuteCount.get()); - } - - @Test - public void testCycleSchedulerWithImmediatelyStart(@Mocked Env env) throws DdlException { - setContext(env); - long startTimestamp = System.currentTimeMillis(); - job.setImmediatelyStart(true); - timerJobManager.registerJob(job); - //consider the time of the first execution and give some buffer time - Awaitility.await().atMost(16, TimeUnit.SECONDS).until(() -> System.currentTimeMillis() - >= startTimestamp + 15000L); - Assertions.assertEquals(3, testExecuteCount.get()); - } - - @Test - public void testOneTimeJob(@Mocked Env env) throws DdlException { - setContext(env); - - long startTimestamp = System.currentTimeMillis() + 3000L; - job.setIntervalMs(0L); - job.setStartTimeMs(startTimestamp); - job.setJobType(JobType.ONE_TIME); - timerJobManager.registerJob(job); - //consider the time of the first execution and give some buffer time - Awaitility.await().atMost(14, TimeUnit.SECONDS).until(() -> System.currentTimeMillis() - >= startTimestamp + 7000L); - Assertions.assertEquals(1, testExecuteCount.get()); - } - - @AfterEach - public void after() throws IOException { - timerJobManager.close(); - } - - class TestExecutor implements JobExecutor { - - @Override - public ExecutorResult execute(Job job, String dataContext) throws JobException { - log.info("test execute count:{}", testExecuteCount.incrementAndGet()); - return new ExecutorResult<>(true, true, null, ""); - } - } -}