diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-JOB.md b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-JOB.md index d89e0aad0d..f173cfee93 100644 --- a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-JOB.md +++ b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-JOB.md @@ -76,7 +76,7 @@ CREATE JOB my_job ON SCHEDULE EVERY 1 MINUTE DO INSERT INTO db1.tbl1 SELECT * FR This statement means to create a job named my_job to be executed every minute, and the operation performed is to import the data in db2.tbl2 into db1.tbl1. -The SCHEDULER statement is used to define the execution time, frequency and duration of the job, which can specify a one-time job or a periodic job. +The SCHEDULE statement is used to define the execution time, frequency and duration of the job, which can specify a one-time job or a periodic job. - AT timestamp For one-time events, it specifies that the event is only executed once at a given date and time timestamp, which must contain the date and time @@ -119,7 +119,7 @@ CREATE JOB my_job ON SCHEDULE EVERY 1 DAY STARTS '2020-01-01 00:00:00' DO INSERT Create a periodic Job, which will start to execute at 2020-01-01 00:00:00, and execute once a day. The operation performed is to import the data in db2.tbl2 into db1.tbl1. This Job will be executed in 2020 Ends at -01-01 00:10:00. ```sql -CREATE JOB my_job ON SCHEDULER EVERY 1 DAY STARTS '2020-01-01 00:00:00' ENDS '2020-01-01 00:10:00' DO INSERT INTO db1.tbl1 SELECT * FROM db2.tbl2 create_time >= days_add (now(),-1); +CREATE JOB my_job ON SCHEDULE EVERY 1 DAY STARTS '2020-01-01 00:00:00' ENDS '2020-01-01 00:10:00' DO INSERT INTO db1.tbl1 SELECT * FROM db2.tbl2 create_time >= days_add (now(),-1); ``` ### Keywords diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-JOB.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-JOB.md index 1202bbff42..70a90f6638 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-JOB.md +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-JOB.md @@ -39,7 +39,7 @@ Doris Job 是根据既定计划运行的任务,用于在特定时间或指定 Job 有两种类型:`ONE_TIME` 和 `RECURRING`。其中 `ONE_TIME` 类型的 Job 会在指定的时间点触发,它主要用于一次性任务,而 `RECURRING` 类型的 Job 会在指定的时间间隔内循环触发, 此方式主要用于周期性执行的任务。 `RECURRING` 类型的 Job 可指定开始时间,结束时间,即 `STARTS\ENDS`, 如果不指定开始时间,则默认首次执行时间为当前时间 + 一次调度周期。如果指定结束时间,则 task 执行完成如果达到结束时间(或超过,或下次执行周期会超过结束时间)则更新为FINISHED状态,此时不会再产生 Task。 -JOB 共4种状态(`RUNNING`,`STOPPED`,`PAUSED`,`FINISHED`,),初始状态为RUNNING,RUNNING状态的JOB会根据既定的调度周期去生成 TASK 执行,Job 执行完成达到结束时间则状态变更为 `FINISHED`. +JOB 共4种状态(`RUNNING`,`STOPPED`,`PAUSED`,`FINISHED`,),初始状态为RUNNING,RUNNING 状态的JOB会根据既定的调度周期去生成 TASK 执行,Job 执行完成达到结束时间则状态变更为 `FINISHED`. RUNNING 状态的JOB 可以被 pause,即暂停,此时不会再生成 Task。 @@ -82,7 +82,7 @@ interval: - 关键字 CREATE JOB 加上作业名称,它在一个 db 中标识唯一事件。 - ON SCHEDULE 子句,它指定了 Job 作业的类型和触发时间以及频率。 -- DO 子句,它指定了 Job 作业触发时需要执行的操作。 +- DO 子句,它指定了 Job 作业触发时需要执行的操作, 即一条 SQL 语句。 这是一个最简单的例子: @@ -92,7 +92,7 @@ CREATE JOB my_job ON SCHEDULE EVERY 1 MINUTE DO INSERT INTO db1.tbl1 SELECT * FR 该语句表示创建一个名为 my_job 的作业,每分钟执行一次,执行的操作是将 db2.tbl2 中的数据导入到 db1.tbl1 中。 -SCHEDULER 语句用于定义作业的执行时间,频率以及持续时间,它可以指定一次性作业或者周期性作业。 +SCHEDULE 语句用于定义作业的执行时间,频率以及持续时间,它可以指定一次性作业或者周期性作业。 - AT timestamp 用于一次性事件,它指定事件仅在 给定的日期和时间执行一次 timestamp,该日期和时间必须包含日期和时间 @@ -135,7 +135,7 @@ CREATE JOB my_job ON SCHEDULE EVERY 1 DAY STARTS '2020-01-01 00:00:00' DO INSERT 创建一个周期性的 Job,它会在 2020-01-01 00:00:00 时开始执行,每天执行一次,执行的操作是将 db2.tbl2 中的数据导入到 db1.tbl1 中,该 Job 在 2020-01-01 00:10:00 时结束。 ```sql -CREATE JOB my_job ON SCHEDULER EVERY 1 DAY STARTS '2020-01-01 00:00:00' ENDS '2020-01-01 00:10:00' DO INSERT INTO db1.tbl1 SELECT * FROM db2.tbl2 create_time >= days_add(now(),-1); +CREATE JOB my_job ON SCHEDULE EVERY 1 DAY STARTS '2020-01-01 00:00:00' ENDS '2020-01-01 00:10:00' DO INSERT INTO db1.tbl1 SELECT * FROM db2.tbl2 create_time >= days_add(now(),-1); ``` ### INSERT JOB 目前仅支持 ***INSERT 内表*** diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 1e474556b4..1be6d99e25 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -569,7 +569,7 @@ terminal String KW_ROWS, KW_S3, KW_SAMPLE, - KW_SCHEDULER, + KW_SCHEDULE, KW_SCHEMA, KW_SCHEMAS, KW_SECOND, @@ -2556,17 +2556,17 @@ resource_desc ::= :} ; create_job_stmt ::= - KW_CREATE KW_JOB job_label:jobLabel KW_ON KW_SCHEDULER KW_EVERY INTEGER_LITERAL:time_interval ident:time_unit opt_job_starts:startsTime opt_job_ends:endsTime opt_comment:comment KW_DO stmt:executeSql + KW_CREATE KW_JOB job_label:jobLabel KW_ON KW_SCHEDULE KW_EVERY INTEGER_LITERAL:time_interval ident:time_unit opt_job_starts:startsTime opt_job_ends:endsTime opt_comment:comment KW_DO stmt:executeSql {: CreateJobStmt stmt = new CreateJobStmt(jobLabel,org.apache.doris.job.base.JobExecuteType.RECURRING,null,time_interval,time_unit, startsTime, endsTime,comment,executeSql); RESULT = stmt; :} -/* support in future | KW_CREATE KW_JOB job_label:jobLabel KW_ON KW_SCHEDULER KW_STREAMING KW_AT STRING_LITERAL:atTime opt_comment:comment KW_DO stmt:executeSql +/* support in future | KW_CREATE KW_JOB job_label:jobLabel KW_ON KW_SCHEDULE KW_STREAMING KW_AT STRING_LITERAL:atTime opt_comment:comment KW_DO stmt:executeSql {: CreateJobStmt stmt = new CreateJobStmt(jobLabel,org.apache.doris.job.base.JobExecuteType.STREAMING,atTime,null,null,null,null,comment,executeSql); RESULT = stmt; :} */ - | KW_CREATE KW_JOB job_label:jobLabel KW_ON KW_SCHEDULER KW_AT STRING_LITERAL:atTime opt_comment:comment KW_DO stmt:executeSql + | KW_CREATE KW_JOB job_label:jobLabel KW_ON KW_SCHEDULE KW_AT STRING_LITERAL:atTime opt_comment:comment KW_DO stmt:executeSql {: CreateJobStmt stmt = new CreateJobStmt(jobLabel,org.apache.doris.job.base.JobExecuteType.ONE_TIME,atTime,null,null,null,null,comment,executeSql); RESULT = stmt; @@ -7672,7 +7672,7 @@ keyword ::= {: RESULT = id; :} | KW_ROLLBACK:id {: RESULT = id; :} - | KW_SCHEDULER:id + | KW_SCHEDULE:id {: RESULT = id; :} | KW_SCHEMA:id {: RESULT = id; :} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateJobStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateJobStmt.java index 1a44ee4f90..dc2083debc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateJobStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateJobStmt.java @@ -201,9 +201,13 @@ public class CreateJobStmt extends DdlStmt { doStmt.analyze(analyzer); } + /** + * parse execute sql from create job stmt + * Some stmt not implement toSql method,so we need to parse sql from originStmt + */ private String parseExecuteSql(String sql) throws AnalysisException { - sql = sql.toLowerCase(); - int executeSqlIndex = sql.indexOf(" do "); + String lowerCaseSql = sql.toLowerCase(); + int executeSqlIndex = lowerCaseSql.indexOf(" do "); String executeSql = sql.substring(executeSqlIndex + 4).trim(); if (StringUtils.isBlank(executeSql)) { throw new AnalysisException("execute sql has invalid format"); diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex b/fe/fe-core/src/main/jflex/sql_scanner.flex index 16c4ae9500..d9bbd93859 100644 --- a/fe/fe-core/src/main/jflex/sql_scanner.flex +++ b/fe/fe-core/src/main/jflex/sql_scanner.flex @@ -422,7 +422,7 @@ import org.apache.doris.qe.SqlModeHelper; keywordMap.put("rows", new Integer(SqlParserSymbols.KW_ROWS)); keywordMap.put("s3", new Integer(SqlParserSymbols.KW_S3)); keywordMap.put("schema", new Integer(SqlParserSymbols.KW_SCHEMA)); - keywordMap.put("scheduler", new Integer(SqlParserSymbols.KW_SCHEDULER)); + keywordMap.put("schedule", new Integer(SqlParserSymbols.KW_SCHEDULE)); keywordMap.put("schemas", new Integer(SqlParserSymbols.KW_SCHEMAS)); keywordMap.put("second", new Integer(SqlParserSymbols.KW_SECOND)); keywordMap.put("select", new Integer(SqlParserSymbols.KW_SELECT)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateJobStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateJobStmtTest.java index f0e800cd6f..fd4edeb0a8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateJobStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateJobStmtTest.java @@ -29,16 +29,16 @@ public class CreateJobStmtTest { @Test public void createOnceTimeJobStmt() throws Exception { - String sql = "CREATE JOB job1 ON SCHEDULER AT \"2023-02-15\" DO SELECT * FROM `address` ;"; + String sql = "CREATE JOB job1 ON SCHEDULE AT \"2023-02-15\" DO SELECT * FROM `address` ;"; CreateJobStmt jobStmt = sqlParse(sql); System.out.println(jobStmt.getDoStmt().toSql()); Assertions.assertEquals("SELECT * FROM `address`", jobStmt.getDoStmt().toSql()); - String badExecuteSql = "CREATE JOB job1 ON SCHEDULER AT \"2023-02-15\" DO selects * from address ;"; + String badExecuteSql = "CREATE JOB job1 ON SCHEDULE AT \"2023-02-15\" DO selects * from address ;"; Assertions.assertThrows(AnalysisException.class, () -> { sqlParse(badExecuteSql); }); - String badSql = "CREATE JOB job1 ON SCHEDULER AT \"2023-02-15\" STARTS \"2023-02-15\" DO selects * from address ;"; + String badSql = "CREATE JOB job1 ON SCHEDULE AT \"2023-02-15\" STARTS \"2023-02-15\" DO selects * from address ;"; Assertions.assertThrows(AnalysisException.class, () -> { sqlParse(badSql); }); @@ -54,23 +54,23 @@ public class CreateJobStmtTest { @Test public void createCycleJob() throws Exception { - String sql = "CREATE JOB job1 ON SCHEDULER EVERY 1 SECOND STARTS \"2023-02-15\" DO SELECT * FROM `address` ;"; + String sql = "CREATE JOB job1 ON SCHEDULE EVERY 1 SECOND STARTS \"2023-02-15\" DO SELECT * FROM `address` ;"; CreateJobStmt jobStmt = sqlParse(sql); Assertions.assertEquals("SELECT * FROM `address`", jobStmt.getDoStmt().toSql()); - sql = "CREATE JOB job1 ON SCHEDULER EVERY 1 SECOND ENDS \"2023-02-15\" DO SELECT * FROM `address` ;"; + sql = "CREATE JOB job1 ON SCHEDULE EVERY 1 SECOND ENDS \"2023-02-15\" DO SELECT * FROM `address` ;"; jobStmt = sqlParse(sql); Assertions.assertEquals("SELECT * FROM `address`", jobStmt.getDoStmt().toSql()); - sql = "CREATE JOB job1 ON SCHEDULER EVERY 1 SECOND STARTS \"2023-02-15\" ENDS \"2023-02-16\" DO SELECT * FROM `address` ;"; + sql = "CREATE JOB job1 ON SCHEDULE EVERY 1 SECOND STARTS \"2023-02-15\" ENDS \"2023-02-16\" DO SELECT * FROM `address` ;"; jobStmt = sqlParse(sql); Assertions.assertEquals("SELECT * FROM `address`", jobStmt.getDoStmt().toSql()); - sql = "CREATE JOB job1 ON SCHEDULER EVERY 1 SECOND DO SELECT * FROM `address` ;"; + sql = "CREATE JOB job1 ON SCHEDULE EVERY 1 SECOND DO SELECT * FROM `address` ;"; jobStmt = sqlParse(sql); Assertions.assertEquals("SELECT * FROM `address`", jobStmt.getDoStmt().toSql()); - String badExecuteSql = "CREATE JOB job1 ON SCHEDULER AT \"2023-02-15\" DO selects * from address ;"; + String badExecuteSql = "CREATE JOB job1 ON SCHEDULE AT \"2023-02-15\" DO selects * from address ;"; Assertions.assertThrows(AnalysisException.class, () -> { sqlParse(badExecuteSql); }); - String badSql = "CREATE JOB job1 ON SCHEDULER AT \"2023-02-15\" STARTS \"2023-02-15\" DO selects * from address ;"; + String badSql = "CREATE JOB job1 ON SCHEDULE AT \"2023-02-15\" STARTS \"2023-02-15\" DO selects * from address ;"; Assertions.assertThrows(AnalysisException.class, () -> { sqlParse(badSql); }); diff --git a/regression-test/suites/job_p0/test_base_insert_job.groovy b/regression-test/suites/job_p0/test_base_insert_job.groovy index 52d627d6a0..035def7a2d 100644 --- a/regression-test/suites/job_p0/test_base_insert_job.groovy +++ b/regression-test/suites/job_p0/test_base_insert_job.groovy @@ -21,7 +21,7 @@ import java.time.Instant; import java.time.ZoneId; suite("test_base_insert_job") { - def tableName = "t_test_base_insert_job" + def tableName = "t_test_BASE_inSert_job" def jobName = "insert_recovery_test_base_insert_job" sql """drop table if exists `${tableName}` force""" sql """ @@ -42,7 +42,7 @@ suite("test_base_insert_job") { ); """ sql """ - CREATE JOB ${jobName} ON SCHEDULER every 1 second comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213'); + CREATE JOB ${jobName} ON SCHEDULE every 1 second comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213'); """ Thread.sleep(2500) def jobs = sql """select * from ${tableName}""" @@ -74,13 +74,19 @@ suite("test_base_insert_job") { def dataCount = sql """select count(*) from ${tableName}""" assert dataCount.get(0).get(0) == 0 sql """ - CREATE JOB ${jobName} ON SCHEDULER at '${startTime}' comment 'test for test&68686781jbjbhj//ncsa' DO insert into ${tableName} values ('2023-07-19', sleep(10000), 1001); + CREATE JOB ${jobName} ON SCHEDULE at '${startTime}' comment 'test for test&68686781jbjbhj//ncsa' DO insert into ${tableName} values ('2023-07-19', sleep(10000), 1001); """ Thread.sleep(25000) - def onceJob = sql """select id from jobs("type"="insert") where Name='${jobName}'""" + def onceJob = sql """select id,ExecuteSql from jobs("type"="insert") where Name='${jobName}'""" assert onceJob.size() == 1 def onceJobId= onceJob.get(0).get(0); + def onceJobSql= onceJob.get(0).get(1); + println onceJobSql + def assertSql = "insert into ${tableName} values (\'2023-07-19\', sleep(10000), 1001);" + println 'hhh' + println assertSql + assert onceJobSql == assertSql // test cancel task def datas = sql """select status,taskid from tasks("type"="insert") where jobid= ${onceJobId}""" println datas @@ -105,7 +111,7 @@ suite("test_base_insert_job") { try{ sql """ - CREATE JOB ${jobName} ON SCHEDULER at '${startTime}' comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213'); + CREATE JOB ${jobName} ON SCHEDULE at '${startTime}' comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213'); """ } catch (Exception e) { assert e.getMessage().contains("startTimeMs must be greater than current time") @@ -115,7 +121,7 @@ suite("test_base_insert_job") { """ try{ sql """ - CREATE JOB test_one_time_error_starts ON SCHEDULER at '2023-11-13 14:18:07' comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213'); + CREATE JOB test_one_time_error_starts ON SCHEDULE at '2023-11-13 14:18:07' comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213'); """ } catch (Exception e) { assert e.getMessage().contains("startTimeMs must be greater than current time") @@ -125,7 +131,7 @@ suite("test_base_insert_job") { """ try{ sql """ - CREATE JOB test_error_starts ON SCHEDULER every 1 second ends '2023-11-13 14:18:07' comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213'); + CREATE JOB test_error_starts ON SCHEDULE every 1 second ends '2023-11-13 14:18:07' comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213'); """ } catch (Exception e) { assert e.getMessage().contains("end time cannot be less than start time") @@ -136,7 +142,7 @@ suite("test_base_insert_job") { """ try{ sql """ - CREATE JOB test_error_starts ON SCHEDULER every 1 years ends '2023-11-13 14:18:07' comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213'); + CREATE JOB test_error_starts ON SCHEDULE every 1 years ends '2023-11-13 14:18:07' comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213'); """ } catch (Exception e) { assert e.getMessage().contains("interval time unit can not be years")