diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java index c2b654e23f..5a7ff739ca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java @@ -93,8 +93,10 @@ public class MaterializedViewHandler extends AlterHandler { private static final Logger LOG = LogManager.getLogger(MaterializedViewHandler.class); public static final String NEW_STORAGE_FORMAT_INDEX_NAME_PREFIX = "__v2_"; + public static int scheduler_interval_millisecond = 333; + public MaterializedViewHandler() { - super("materialized view"); + super("materialized view", scheduler_interval_millisecond); } // for batch submit rollup job, tableId -> jobId diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/BatchRollupJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/BatchRollupJobTest.java index 987d90215b..ac3522ad83 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/BatchRollupJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/BatchRollupJobTest.java @@ -92,7 +92,7 @@ public class BatchRollupJobTest { while (!alterJobV2.getJobState().isFinalState()) { System.out.println( "rollup job " + alterJobV2.getJobId() + " is running. state: " + alterJobV2.getJobState()); - Thread.sleep(5000); + Thread.sleep(1000); } System.out.println("rollup job " + alterJobV2.getJobId() + " is done. state: " + alterJobV2.getJobState()); Assert.assertEquals(AlterJobV2.JobState.FINISHED, alterJobV2.getJobState()); @@ -142,7 +142,7 @@ public class BatchRollupJobTest { while (!alterJobV2.getJobState().isFinalState()) { System.out.println( "rollup job " + alterJobV2.getJobId() + " is running. state: " + alterJobV2.getJobState()); - Thread.sleep(5000); + Thread.sleep(1000); } System.out.println("rollup job " + alterJobV2.getJobId() + " is done. state: " + alterJobV2.getJobState()); Assert.assertEquals(AlterJobV2.JobState.FINISHED, alterJobV2.getJobState()); diff --git a/regression-test/data/materialized_view_p0/ut/testProjectionMV1/testProjectionMV1.out b/regression-test/data/materialized_view_p0/ut/testProjectionMV1/testProjectionMV1.out new file mode 100644 index 0000000000..ba455b7e85 --- /dev/null +++ b/regression-test/data/materialized_view_p0/ut/testProjectionMV1/testProjectionMV1.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 + +-- !select_mv -- +1 1 +1 1 +2 2 + diff --git a/regression-test/data/materialized_view_p0/ut/testProjectionMV2/testProjectionMV2.out b/regression-test/data/materialized_view_p0/ut/testProjectionMV2/testProjectionMV2.out new file mode 100644 index 0000000000..934d3208d4 --- /dev/null +++ b/regression-test/data/materialized_view_p0/ut/testProjectionMV2/testProjectionMV2.out @@ -0,0 +1,14 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 + +-- !select_mv -- +2 +2 + +-- !select_base -- +a +a + diff --git a/regression-test/data/materialized_view_p0/ut/testProjectionMV3/testProjectionMV3.out b/regression-test/data/materialized_view_p0/ut/testProjectionMV3/testProjectionMV3.out new file mode 100644 index 0000000000..9537036f41 --- /dev/null +++ b/regression-test/data/materialized_view_p0/ut/testProjectionMV3/testProjectionMV3.out @@ -0,0 +1,14 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 + +-- !select_mv -- +2 a +2 a + +-- !select_mv2 -- +a +a + diff --git a/regression-test/data/materialized_view_p0/ut/testProjectionMV4/testProjectionMV4.out b/regression-test/data/materialized_view_p0/ut/testProjectionMV4/testProjectionMV4.out new file mode 100644 index 0000000000..45a962ed59 --- /dev/null +++ b/regression-test/data/materialized_view_p0/ut/testProjectionMV4/testProjectionMV4.out @@ -0,0 +1,12 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_star -- +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 + +-- !select_mv -- +b + +-- !select_base -- +2 + diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/CreateMVAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/CreateMVAction.groovy new file mode 100644 index 0000000000..b4b37df016 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/CreateMVAction.groovy @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.action + +import groovy.transform.stc.ClosureParams +import groovy.transform.stc.FromString +import org.apache.doris.regression.suite.SuiteContext +import org.apache.doris.regression.util.JdbcUtils +import groovy.util.logging.Slf4j +import java.sql.ResultSetMetaData +import java.util.stream.Collectors +import java.sql.Connection + +@Slf4j +class CreateMVAction implements SuiteAction { + private String sql + private String exception + private SuiteContext context + + CreateMVAction(SuiteContext context, String sql) { + this.context = context + this.sql = sql + } + + CreateMVAction(SuiteContext context, String sql, String exception) { + this.context = context + this.sql = sql + this.exception = exception + } + + @Override + void run() { + def result = doRun(sql) + String resultString = result.result + + if (exception != null) { + if (result.exception == null) { + throw new IllegalStateException("Expect exception on create mv, but create success"); + } + def msg = result.exception?.toString() + Assert.assertTrue("Expect exception msg contains '${exception}', but meet '${msg}'", + msg != null && exception != null && msg.contains(exception)) + return + } + + def tryTimes = 0 + def sqlResult = "null" + while (!sqlResult.contains("FINISHED")) { + def tmp = doRun("SHOW ALTER TABLE MATERIALIZED VIEW ORDER BY CreateTime DESC LIMIT 1;") + sqlResult = tmp.result[0] + log.info("result: ${sqlResult}") + if (tryTimes == 60 || sqlResult.contains("CANCELLED")) { + throw new IllegalStateException("MV create check times over limit"); + } + Thread.sleep(1200) + tryTimes++ + } + + Thread.sleep(1000) + } + + ActionResult doRun(String sql) { + Connection conn = context.getConnection() + List> result = null + ResultSetMetaData meta = null + Throwable ex = null + + long startTime = System.currentTimeMillis() + try { + log.info("sql:\n${sql}".toString()) + (result, meta) = JdbcUtils.executeToList(conn, sql) + } catch (Throwable t) { + ex = t + } + long endTime = System.currentTimeMillis() + + return new ActionResult(result, ex, startTime, endTime, meta) + } + + class ActionResult { + List> result + Throwable exception + long startTime + long endTime + ResultSetMetaData meta + + ActionResult(List> result, Throwable exception, long startTime, long endTime, ResultSetMetaData meta) { + this.result = result + this.exception = exception + this.startTime = startTime + this.endTime = endTime + this.meta = meta + } + } +} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 0cbde97baa..c4a1eaf23e 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableList import org.apache.doris.regression.action.BenchmarkAction import org.apache.doris.regression.util.DataUtils import org.apache.doris.regression.util.OutputUtils +import org.apache.doris.regression.action.CreateMVAction import org.apache.doris.regression.action.ExplainAction import org.apache.doris.regression.action.RestoreAction import org.apache.doris.regression.action.StreamLoadAction @@ -271,6 +272,14 @@ class Suite implements GroovyInterceptable { runAction(new ExplainAction(context), actionSupplier) } + void createMV(String sql) { + (new CreateMVAction(context, sql)).run() + } + + void createMV(String sql, String expection) { + (new CreateMVAction(context, sql, expection)).run() + } + void test(Closure actionSupplier) { runAction(new TestAction(context), actionSupplier) } diff --git a/regression-test/suites/materialized_view_p0/ut/testProjectionMV1/testProjectionMV1.groovy b/regression-test/suites/materialized_view_p0/ut/testProjectionMV1/testProjectionMV1.groovy new file mode 100644 index 0000000000..c23b533832 --- /dev/null +++ b/regression-test/suites/materialized_view_p0/ut/testProjectionMV1/testProjectionMV1.groovy @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("testProjectionMV1") { + sql """ DROP TABLE IF EXISTS emps; """ + + sql """ + create table emps ( + time_col date, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" + + createMV("create materialized view emps_mv as select deptno, empid from emps order by deptno;") + + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + + explain { + sql("select * from emps order by empid;") + contains "(emps)" + } + qt_select_star "select * from emps order by empid;" + + + explain { + sql("select empid, deptno from emps order by empid;") + contains "(emps_mv)" + } + qt_select_mv "select empid, deptno from emps order by empid;" +} diff --git a/regression-test/suites/materialized_view_p0/ut/testProjectionMV2/testProjectionMV2.groovy b/regression-test/suites/materialized_view_p0/ut/testProjectionMV2/testProjectionMV2.groovy new file mode 100644 index 0000000000..4b1936f5e0 --- /dev/null +++ b/regression-test/suites/materialized_view_p0/ut/testProjectionMV2/testProjectionMV2.groovy @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("testProjectionMV2") { + sql """ DROP TABLE IF EXISTS emps; """ + + sql """ + create table emps ( + time_col date, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" + + createMV("create materialized view emps_mv as select deptno, empid from emps order by deptno;") + + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + + explain { + sql("select * from emps order by empid;") + contains "(emps)" + } + qt_select_star "select * from emps order by empid;" + + + explain { + sql("select empid + 1 from emps where deptno = 1 order by empid;") + contains "(emps_mv)" + } + qt_select_mv "select empid + 1 from emps where deptno = 1 order by empid;" + + explain { + sql("select name from emps where deptno -1 = 0 order by empid;") + contains "(emps)" + } + qt_select_base "select name from emps where deptno -1 = 0 order by empid;" +} diff --git a/regression-test/suites/materialized_view_p0/ut/testProjectionMV3/testProjectionMV3.groovy b/regression-test/suites/materialized_view_p0/ut/testProjectionMV3/testProjectionMV3.groovy new file mode 100644 index 0000000000..261168e48f --- /dev/null +++ b/regression-test/suites/materialized_view_p0/ut/testProjectionMV3/testProjectionMV3.groovy @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("testProjectionMV3") { + sql """ DROP TABLE IF EXISTS emps; """ + + sql """ + create table emps ( + time_col date, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" + + def result = "null" + + createMV("create materialized view emps_mv as select deptno, empid, name from emps order by deptno;") + + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + + explain { + sql("select * from emps order by empid;") + contains "(emps)" + } + qt_select_star "select * from emps order by empid;" + + + explain { + sql("select empid + 1, name from emps where deptno = 1 order by empid;") + contains "(emps_mv)" + } + qt_select_mv "select empid + 1, name from emps where deptno = 1 order by empid;" + + explain { + sql("select name from emps where deptno -1 = 0 order by empid;") + contains "(emps_mv)" + } + qt_select_mv2 "select name from emps where deptno -1 = 0 order by empid;" +} diff --git a/regression-test/suites/materialized_view_p0/ut/testProjectionMV4/testProjectionMV4.groovy b/regression-test/suites/materialized_view_p0/ut/testProjectionMV4/testProjectionMV4.groovy new file mode 100644 index 0000000000..969e547630 --- /dev/null +++ b/regression-test/suites/materialized_view_p0/ut/testProjectionMV4/testProjectionMV4.groovy @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("testProjectionMV4") { + sql """ DROP TABLE IF EXISTS emps; """ + + sql """ + create table emps ( + time_col date, + empid int, + name varchar, + deptno int, + salary int, + commission int) + partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); + """ + + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" + + def result = "null" + + createMV("create materialized view emps_mv as select name, deptno, salary from emps;") + + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + + explain { + sql("select * from emps order by empid;") + contains "(emps)" + } + qt_select_star "select * from emps order by empid;" + + + explain { + sql("select name from emps where deptno > 1 and salary > 1 order by name;") + contains "(emps_mv)" + } + qt_select_mv "select name from emps where deptno > 1 and salary > 1 order by name;" + + explain { + sql("select empid from emps where deptno > 1 and empid > 1 order by empid;") + contains "(emps)" + } + qt_select_base "select empid from emps where deptno > 1 and empid > 1 order by empid;" +}