From 2a74d9a8c846f4c7387a22ba76616731cb611964 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Mon, 30 Oct 2023 19:49:45 +0800 Subject: [PATCH] [feature](mtmv)(1)remove old mtmv code (#26041) remove old mtmv code,we will implement mtmv in a new way --- fe/fe-core/src/main/cup/sql_parser.cup | 164 ----- .../java/org/apache/doris/alter/Alter.java | 57 -- .../doris/alter/MaterializedViewHandler.java | 14 - .../analysis/AlterMaterializedViewStmt.java | 65 -- .../doris/analysis/ColumnPartitionDesc.java | 135 ---- .../CreateMultiTableMaterializedViewStmt.java | 232 ------- .../doris/analysis/CreateTableStmt.java | 3 +- .../apache/doris/analysis/MVRefreshInfo.java | 91 --- .../MVRefreshIntervalTriggerInfo.java | 63 -- .../doris/analysis/MVRefreshTriggerInfo.java | 77 --- .../analysis/RefreshMaterializedViewStmt.java | 76 --- .../doris/analysis/ShowMTMVJobStmt.java | 143 ----- .../doris/analysis/ShowMTMVTaskStmt.java | 142 ----- .../java/org/apache/doris/catalog/Env.java | 52 -- .../doris/catalog/MaterializedView.java | 129 ---- .../doris/catalog/OlapTableFactory.java | 80 +-- .../java/org/apache/doris/catalog/Table.java | 2 - .../doris/datasource/InternalCatalog.java | 20 +- .../apache/doris/journal/JournalEntity.java | 43 +- .../org/apache/doris/mtmv/MTMVJobFactory.java | 117 ---- .../org/apache/doris/mtmv/MTMVJobManager.java | 422 ------------- .../apache/doris/mtmv/MTMVTaskContext.java | 72 --- .../doris/mtmv/MTMVTaskExecuteParams.java | 36 -- .../apache/doris/mtmv/MTMVTaskExecutor.java | 181 ------ .../doris/mtmv/MTMVTaskExecutorPool.java | 82 --- .../apache/doris/mtmv/MTMVTaskManager.java | 350 ----------- .../apache/doris/mtmv/MTMVTaskProcessor.java | 158 ----- .../java/org/apache/doris/mtmv/MTMVUtils.java | 137 ----- .../doris/mtmv/metadata/ChangeMTMVJob.java | 99 --- .../doris/mtmv/metadata/DropMTMVJob.java | 52 -- .../doris/mtmv/metadata/DropMTMVTask.java | 53 -- .../mtmv/metadata/MTMVCheckpointData.java | 30 - .../apache/doris/mtmv/metadata/MTMVJob.java | 364 ----------- .../apache/doris/mtmv/metadata/MTMVTask.java | 244 -------- .../persist/AlterMultiMaterializedView.java | 61 -- .../org/apache/doris/persist/EditLog.java | 65 +- .../apache/doris/persist/OperationType.java | 9 +- .../doris/persist/meta/MetaPersistMethod.java | 6 - .../persist/meta/PersistMetaModules.java | 2 +- .../java/org/apache/doris/qe/DdlExecutor.java | 9 - .../org/apache/doris/qe/ShowExecutor.java | 49 -- .../MultiTableMaterializedViewTest.java | 577 ------------------ .../apache/doris/mtmv/MTMVJobManagerTest.java | 145 ----- .../doris/mtmv/MTMVTaskExecutorTest.java | 116 ---- .../org/apache/doris/mtmv/MTMVUtilsTest.java | 90 --- .../doris/mtmv/ShowMTMVJobStmtTest.java | 103 ---- .../doris/mtmv/ShowMTMVTaskStmtTest.java | 104 ---- .../data/mtmv_p0/test_create_mtmv.out | 6 - .../data/mtmv_p0/test_refresh_mtmv.out | 6 - .../doris/regression/suite/Suite.groovy | 23 - .../mtmv_p0/ssb/ddl/customer_create.sql | 15 - .../mtmv_p0/ssb/ddl/customer_delete.sql | 1 - .../suites/mtmv_p0/ssb/ddl/date_create.sql | 24 - .../suites/mtmv_p0/ssb/ddl/date_delete.sql | 1 - .../mtmv_p0/ssb/ddl/lineorder_create.sql | 24 - .../mtmv_p0/ssb/ddl/lineorder_delete.sql | 1 - .../mtmv_p0/ssb/ddl/lineorder_flat_create.sql | 45 -- .../mtmv_p0/ssb/ddl/lineorder_flat_delete.sql | 1 - .../suites/mtmv_p0/ssb/ddl/part_create.sql | 16 - .../suites/mtmv_p0/ssb/ddl/part_delete.sql | 1 - .../mtmv_p0/ssb/ddl/ssb_flat_create.sql | 52 -- .../suites/mtmv_p0/ssb/ddl/ssb_q11_create.sql | 12 - .../suites/mtmv_p0/ssb/ddl/ssb_q12_create.sql | 12 - .../suites/mtmv_p0/ssb/ddl/ssb_q13_create.sql | 14 - .../suites/mtmv_p0/ssb/ddl/ssb_q21_create.sql | 15 - .../suites/mtmv_p0/ssb/ddl/ssb_q22_create.sql | 15 - .../suites/mtmv_p0/ssb/ddl/ssb_q23_create.sql | 15 - .../suites/mtmv_p0/ssb/ddl/ssb_q31_create.sql | 21 - .../suites/mtmv_p0/ssb/ddl/ssb_q32_create.sql | 21 - .../suites/mtmv_p0/ssb/ddl/ssb_q33_create.sql | 27 - .../suites/mtmv_p0/ssb/ddl/ssb_q34_create.sql | 26 - .../suites/mtmv_p0/ssb/ddl/ssb_q41_create.sql | 23 - .../suites/mtmv_p0/ssb/ddl/ssb_q42_create.sql | 28 - .../suites/mtmv_p0/ssb/ddl/ssb_q43_create.sql | 24 - .../mtmv_p0/ssb/ddl/supplier_create.sql | 14 - .../mtmv_p0/ssb/ddl/supplier_delete.sql | 1 - .../suites/mtmv_p0/test_alter_mtmv.groovy | 86 --- .../mtmv_p0/test_create_both_mtmv.groovy | 78 --- .../suites/mtmv_p0/test_create_mtmv.groovy | 75 --- .../suites/mtmv_p0/test_refresh_mtmv.groovy | 110 ---- 80 files changed, 35 insertions(+), 6119 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/AlterMaterializedViewStmt.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnPartitionDesc.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMultiTableMaterializedViewStmt.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/MVRefreshInfo.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/MVRefreshIntervalTriggerInfo.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/MVRefreshTriggerInfo.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshMaterializedViewStmt.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/ShowMTMVJobStmt.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/analysis/ShowMTMVTaskStmt.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedView.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobFactory.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskContext.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskExecuteParams.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskExecutor.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskExecutorPool.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskManager.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskProcessor.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtils.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/ChangeMTMVJob.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/DropMTMVJob.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/DropMTMVTask.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/MTMVCheckpointData.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/MTMVJob.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/MTMVTask.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/persist/AlterMultiMaterializedView.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/catalog/MultiTableMaterializedViewTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVJobManagerTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskExecutorTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVUtilsTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/mtmv/ShowMTMVJobStmtTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/mtmv/ShowMTMVTaskStmtTest.java delete mode 100644 regression-test/data/mtmv_p0/test_create_mtmv.out delete mode 100644 regression-test/data/mtmv_p0/test_refresh_mtmv.out delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/customer_create.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/customer_delete.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/date_create.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/date_delete.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/lineorder_create.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/lineorder_delete.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/lineorder_flat_create.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/lineorder_flat_delete.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/part_create.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/part_delete.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/ssb_flat_create.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/ssb_q11_create.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/ssb_q12_create.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/ssb_q13_create.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/ssb_q21_create.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/ssb_q22_create.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/ssb_q23_create.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/ssb_q31_create.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/ssb_q32_create.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/ssb_q33_create.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/ssb_q34_create.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/ssb_q41_create.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/ssb_q42_create.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/ssb_q43_create.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/supplier_create.sql delete mode 100644 regression-test/suites/mtmv_p0/ssb/ddl/supplier_delete.sql delete mode 100644 regression-test/suites/mtmv_p0/test_alter_mtmv.groovy delete mode 100644 regression-test/suites/mtmv_p0/test_create_both_mtmv.groovy delete mode 100644 regression-test/suites/mtmv_p0/test_create_mtmv.groovy delete mode 100644 regression-test/suites/mtmv_p0/test_refresh_mtmv.groovy diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index e33d32ddfa..80a71b7490 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -847,7 +847,6 @@ nonterminal DistributionDesc opt_distribution; nonterminal Integer opt_distribution_number; nonterminal Long opt_field_length; nonterminal KeysDesc opt_keys; -nonterminal KeysDesc opt_mv_keys; nonterminal Long opt_id; nonterminal PartitionKeyDesc partition_key_desc; @@ -955,14 +954,6 @@ nonterminal StorageBackend storage_backend; nonterminal ArrayList opt_lock_tables_list; nonterminal LockTable lock_table; -// mv -nonterminal MVRefreshInfo.BuildMode build_mv; -nonterminal MVRefreshInfo.RefreshMethod opt_refresh_method; -nonterminal MVRefreshTriggerInfo opt_refresh_trigger; -nonterminal MVRefreshInfo opt_mv_refersh_info; -nonterminal PartitionDesc opt_mv_partition; - - precedence nonassoc COMMA; precedence nonassoc STRING_LITERAL; precedence nonassoc KW_COLUMNS; @@ -1234,14 +1225,6 @@ refresh_stmt ::= {: RESULT = new RefreshDbStmt(ctl, db, properties); :} - | KW_REFRESH KW_MATERIALIZED KW_VIEW table_name:mv - {: - RESULT = new RefreshMaterializedViewStmt(mv, MVRefreshInfo.RefreshMethod.COMPLETE); - :} - | KW_REFRESH KW_MATERIALIZED KW_VIEW table_name:mv KW_COMPLETE - {: - RESULT = new RefreshMaterializedViewStmt(mv, MVRefreshInfo.RefreshMethod.COMPLETE); - :} | KW_REFRESH KW_CATALOG ident:catalogName opt_properties:properties {: RESULT = new RefreshCatalogStmt(catalogName, properties); @@ -1396,10 +1379,6 @@ alter_stmt ::= {: RESULT = new AlterPolicyStmt(policyName, properties); :} - | KW_ALTER KW_MATERIALIZED KW_VIEW table_name:mv opt_mv_refersh_info:refreshInfo - {: - RESULT = new AlterMaterializedViewStmt(mv, refreshInfo); - :} | KW_ALTER KW_USER opt_if_exists:ifExists grant_user:user @@ -1733,97 +1712,6 @@ opt_intermediate_type ::= :} ; -build_mv ::= - KW_BUILD KW_IMMEDIATE - {: - RESULT = MVRefreshInfo.BuildMode.IMMEDIATE; - :} - | KW_BUILD KW_DEFERRED - {: - RESULT = MVRefreshInfo.BuildMode.DEFERRED; - :} - ; - -opt_refresh_method ::= - {: - RESULT = null; - :} - | KW_FAST - {: - RESULT = MVRefreshInfo.RefreshMethod.FAST; - :} - | KW_COMPLETE - {: - RESULT = MVRefreshInfo.RefreshMethod.COMPLETE; - :} - | KW_FORCE - {: - RESULT = MVRefreshInfo.RefreshMethod.FORCE; - :} - ; - -opt_refresh_trigger ::= - {: - RESULT = new MVRefreshTriggerInfo(MVRefreshInfo.RefreshTrigger.DEMAND); - :} - | KW_ON KW_DEMAND - {: - RESULT = new MVRefreshTriggerInfo(MVRefreshInfo.RefreshTrigger.DEMAND); - :} - | KW_ON KW_COMMIT - {: - RESULT = new MVRefreshTriggerInfo(MVRefreshInfo.RefreshTrigger.COMMIT); - :} - | KW_START KW_WITH ident:start_time - {: - RESULT = new MVRefreshTriggerInfo(new MVRefreshIntervalTriggerInfo(start_time, -1, null)); - :} - | KW_START KW_WITH STRING_LITERAL:start_time KW_NEXT INTEGER_LITERAL:interval time_unit:unit - {: - RESULT = new MVRefreshTriggerInfo(new MVRefreshIntervalTriggerInfo(start_time, interval, unit)); - :} - | KW_NEXT INTEGER_LITERAL:interval time_unit:unit - {: - RESULT = new MVRefreshTriggerInfo(new MVRefreshIntervalTriggerInfo(null, interval, unit)); - :} - ; - -opt_mv_refersh_info ::= - {: - RESULT = new MVRefreshInfo(true); - :} - | KW_REFRESH opt_refresh_method:refreshMethod opt_refresh_trigger:refreshTrigger - {: - RESULT = new MVRefreshInfo(refreshMethod, refreshTrigger); - :} - | KW_NEVER KW_REFRESH - {: - RESULT = new MVRefreshInfo(true); - :} - ; - -opt_mv_keys ::= - {: - RESULT = null; - :} - | KW_KEY LPAREN ident_list:keys RPAREN - {: - RESULT = new KeysDesc(KeysType.DUP_KEYS, keys); - :} - ; - -opt_mv_partition ::= - /* Empty: no partition */ - {: - RESULT = null; - :} - /* Column partition */ - | KW_PARTITION KW_BY LPAREN column_ref_list:columns RPAREN - {: - RESULT = new ColumnPartitionDesc(columns); - :} - ; - // Create Statement create_stmt ::= /* Database */ @@ -1970,15 +1858,6 @@ create_stmt ::= {: RESULT = new CreateMaterializedViewStmt(mvName, selectStmt, properties); :} - | KW_CREATE KW_MATERIALIZED KW_VIEW ident:mvName build_mv:buildMethod - opt_mv_refersh_info:refreshInfo - opt_mv_keys:keyDesc - opt_distribution:distributionDesc - opt_properties:properties - KW_AS query_stmt:query - {: - RESULT = new CreateMultiTableMaterializedViewStmt(mvName, buildMethod, refreshInfo, keyDesc, null, distributionDesc, properties, query); - :} | KW_CREATE KW_INDEX opt_if_not_exists:ifNotExists ident:indexName KW_ON table_name:tableName LPAREN ident_list:cols RPAREN opt_index_type:indexType opt_properties:properties opt_comment:comment {: RESULT = new AlterTableStmt(tableName, Lists.newArrayList(new CreateIndexClause(tableName, new IndexDef(indexName, ifNotExists, cols, indexType, properties, comment), false))); @@ -2855,49 +2734,6 @@ show_routine_load_task_stmt ::= :} ; -show_mtmv_stmt ::= - KW_SHOW KW_MTMV KW_JOB - {: - RESULT = new ShowMTMVJobStmt(); - :} - | KW_SHOW KW_MTMV KW_JOB KW_FOR ident:jobName - {: - RESULT = new ShowMTMVJobStmt(jobName); - :} - | KW_SHOW KW_MTMV KW_JOB KW_FROM ident:dbName - {: - RESULT = new ShowMTMVJobStmt(dbName, null); - :} - | KW_SHOW KW_MTMV KW_JOB KW_ON table_name:tbl - {: - RESULT = new ShowMTMVJobStmt(null, tbl); - :} - | KW_SHOW KW_MTMV KW_JOB KW_FROM ident:dbName KW_ON table_name:tbl - {: - RESULT = new ShowMTMVJobStmt(dbName, tbl); - :} - | KW_SHOW KW_MTMV KW_TASK - {: - RESULT = new ShowMTMVTaskStmt(); - :} - | KW_SHOW KW_MTMV KW_TASK KW_FOR ident:taskId - {: - RESULT = new ShowMTMVTaskStmt(taskId); - :} - | KW_SHOW KW_MTMV KW_TASK KW_FROM ident:dbName - {: - RESULT = new ShowMTMVTaskStmt(dbName, null); - :} - | KW_SHOW KW_MTMV KW_TASK KW_ON table_name:tbl - {: - RESULT = new ShowMTMVTaskStmt(null, tbl); - :} - | KW_SHOW KW_MTMV KW_TASK KW_FROM ident:dbName KW_ON table_name:tbl - {: - RESULT = new ShowMTMVTaskStmt(dbName, tbl); - :} - ; - show_create_routine_load_stmt ::= KW_SHOW KW_CREATE KW_ROUTINE KW_LOAD KW_FOR job_label:jobLabel {: diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index 95c9b46f8b..965ed055df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -25,12 +25,10 @@ import org.apache.doris.analysis.AlterTableStmt; import org.apache.doris.analysis.AlterViewStmt; import org.apache.doris.analysis.ColumnRenameClause; import org.apache.doris.analysis.CreateMaterializedViewStmt; -import org.apache.doris.analysis.CreateMultiTableMaterializedViewStmt; import org.apache.doris.analysis.DropMaterializedViewStmt; import org.apache.doris.analysis.DropPartitionClause; import org.apache.doris.analysis.DropPartitionFromIndexClause; import org.apache.doris.analysis.DropTableStmt; -import org.apache.doris.analysis.MVRefreshInfo.RefreshMethod; import org.apache.doris.analysis.ModifyColumnCommentClause; import org.apache.doris.analysis.ModifyDistributionClause; import org.apache.doris.analysis.ModifyEngineClause; @@ -38,7 +36,6 @@ import org.apache.doris.analysis.ModifyPartitionClause; import org.apache.doris.analysis.ModifyTableCommentClause; import org.apache.doris.analysis.ModifyTablePropertiesClause; import org.apache.doris.analysis.PartitionRenameClause; -import org.apache.doris.analysis.RefreshMaterializedViewStmt; import org.apache.doris.analysis.ReplacePartitionClause; import org.apache.doris.analysis.ReplaceTableClause; import org.apache.doris.analysis.RollupRenameClause; @@ -48,7 +45,6 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DataProperty; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.MaterializedView; import org.apache.doris.catalog.MysqlTable; import org.apache.doris.catalog.OdbcTable; import org.apache.doris.catalog.OlapTable; @@ -66,9 +62,6 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.DynamicPartitionUtil; import org.apache.doris.common.util.MetaLockUtils; import org.apache.doris.common.util.PropertyAnalyzer; -import org.apache.doris.mtmv.MTMVJobFactory; -import org.apache.doris.mtmv.metadata.MTMVJob; -import org.apache.doris.persist.AlterMultiMaterializedView; import org.apache.doris.persist.AlterViewInfo; import org.apache.doris.persist.BatchModifyPartitionsInfo; import org.apache.doris.persist.ModifyCommentOperationLog; @@ -126,11 +119,6 @@ public class Alter { ((MaterializedViewHandler) materializedViewHandler).processCreateMaterializedView(stmt, db, olapTable); } - public void processCreateMultiTableMaterializedView(CreateMultiTableMaterializedViewStmt stmt) - throws UserException { - ((MaterializedViewHandler) materializedViewHandler).processCreateMultiTablesMaterializedView(stmt); - } - public void processDropMaterializedView(DropMaterializedViewStmt stmt) throws DdlException, MetaNotFoundException { if (!stmt.isForMTMV() && stmt.getTableName() == null) { throw new DdlException("Drop materialized view without table name is unsupported : " + stmt.toSql()); @@ -154,16 +142,6 @@ public class Alter { } } - public void processRefreshMaterializedView(RefreshMaterializedViewStmt stmt) - throws DdlException, MetaNotFoundException { - if (stmt.getRefreshMethod() != RefreshMethod.COMPLETE) { - throw new DdlException("Now only support REFRESH COMPLETE."); - } - String db = stmt.getMvName().getDb(); - String tbl = stmt.getMvName().getTbl(); - Env.getCurrentEnv().getMTMVJobManager().refreshMTMV(db, tbl); - } - private boolean processAlterOlapTable(AlterTableStmt stmt, OlapTable olapTable, List alterClauses, final String clusterName, Database db) throws UserException { if (olapTable.getDataSortInfo() != null @@ -537,41 +515,6 @@ public class Alter { } } - public void processAlterMaterializedView(AlterMultiMaterializedView alterView, boolean isReplay) - throws UserException { - TableName tbl = alterView.getMvName(); - MaterializedView olapTable = null; - try { - // 1. check mv exist - Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(tbl.getDb()); - olapTable = (MaterializedView) db.getTableOrMetaException(tbl.getTbl(), TableType.MATERIALIZED_VIEW); - - // 2. drop old job and kill the associated tasks - Env.getCurrentEnv().getMTMVJobManager().dropJobByName(tbl.getDb(), tbl.getTbl(), isReplay); - - // 3. overwrite the refresh info in the memory of fe. - olapTable.writeLock(); - olapTable.setRefreshInfo(alterView.getInfo()); - - // 4. log it and replay it in the follower - if (!isReplay) { - Env.getCurrentEnv().getEditLog().logAlterMTMV(alterView); - // 5. master node generate new jobs - if (MTMVJobFactory.isGenerateJob(olapTable)) { - List jobs = MTMVJobFactory.buildJob(olapTable, db.getFullName()); - for (MTMVJob job : jobs) { - Env.getCurrentEnv().getMTMVJobManager().createJob(job, false); - } - LOG.info("Alter mv success with new mv job created."); - } - } - } finally { - if (olapTable != null) { - olapTable.writeUnlock(); - } - } - } - // entry of processing replace table private void processReplaceTable(Database db, OlapTable origTable, List alterClauses) throws UserException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java index d75327a626..9c55d75534 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java @@ -23,7 +23,6 @@ import org.apache.doris.analysis.CancelAlterTableStmt; import org.apache.doris.analysis.CancelStmt; import org.apache.doris.analysis.CastExpr; import org.apache.doris.analysis.CreateMaterializedViewStmt; -import org.apache.doris.analysis.CreateMultiTableMaterializedViewStmt; import org.apache.doris.analysis.DropMaterializedViewStmt; import org.apache.doris.analysis.DropRollupClause; import org.apache.doris.analysis.MVColumnItem; @@ -44,7 +43,6 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.Replica.ReplicaState; import org.apache.doris.catalog.Table; -import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.TabletInvertedIndex; import org.apache.doris.catalog.TabletMeta; @@ -53,7 +51,6 @@ import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.MetaNotFoundException; -import org.apache.doris.common.UserException; import org.apache.doris.common.util.IdGeneratorUtil; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.PropertyAnalyzer; @@ -1292,15 +1289,4 @@ public class MaterializedViewHandler extends AlterHandler { public Map> getTableRunningJobMap() { return tableRunningJobMap; } - - public void processCreateMultiTablesMaterializedView(CreateMultiTableMaterializedViewStmt addMVClause) - throws UserException { - Map olapTables = addMVClause.getTables(); - try { - olapTables.values().forEach(TableIf::readLock); - Env.getCurrentEnv().createTable(addMVClause); - } finally { - olapTables.values().forEach(TableIf::readUnlock); - } - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterMaterializedViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterMaterializedViewStmt.java deleted file mode 100644 index b0ecc5261a..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterMaterializedViewStmt.java +++ /dev/null @@ -1,65 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.analysis; - -import org.apache.doris.catalog.Env; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.ErrorCode; -import org.apache.doris.common.ErrorReport; -import org.apache.doris.common.UserException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; - -public class AlterMaterializedViewStmt extends DdlStmt { - private TableName mvName; - private MVRefreshInfo info; - - public AlterMaterializedViewStmt(TableName mvName, MVRefreshInfo info) { - this.mvName = mvName; - this.info = info; - } - - public TableName getTable() { - return mvName; - } - - public MVRefreshInfo getRefreshInfo() { - return info; - } - - @Override - public void analyze(Analyzer analyzer) throws AnalysisException, UserException { - super.analyze(analyzer); - mvName.analyze(analyzer); - if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ConnectContext.get(), mvName.getDb(), mvName.getTbl(), - PrivPredicate.ALTER)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "MATERIALIZED VIEW", - ConnectContext.get().getQualifiedUser(), - ConnectContext.get().getRemoteIP(), - mvName.getDb() + ": " + mvName.getTbl()); - } - } - - @Override - public String toSql() { - StringBuilder sb = new StringBuilder(); - sb.append("ALTER MATERIALIZED VIEW ").append(mvName.toSql()).append(" ").append(info.toString()); - return sb.toString(); - - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnPartitionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnPartitionDesc.java deleted file mode 100644 index ab938c88bb..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnPartitionDesc.java +++ /dev/null @@ -1,135 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.analysis; - -import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.PartitionInfo; -import org.apache.doris.catalog.Table; -import org.apache.doris.catalog.TableIf; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.DdlException; - -import com.google.common.collect.Lists; - -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * This class is used for {@link org.apache.doris.catalog.MaterializedView}. - * When we create a materialized view for multiple tables, we can specify the partition of the view - * which is as consistent as one of base tables. - */ -public class ColumnPartitionDesc extends PartitionDesc { - private final List columns; - - public ColumnPartitionDesc(List columns) - throws AnalysisException { - this.columns = columns; - } - - public void analyze(Analyzer analyzer, CreateMultiTableMaterializedViewStmt stmt) throws AnalysisException { - for (SlotRef column : columns) { - column.analyze(analyzer); - } - Table olapTable = matchTable(stmt.getTables()); - PartitionDesc partitionDesc = ((OlapTable) olapTable).getPartitionInfo().toPartitionDesc((OlapTable) olapTable); - type = partitionDesc.getType(); - partitionColNames = toMVPartitionColumnNames(olapTable.getName(), partitionDesc.getPartitionColNames(), - stmt.getQueryStmt()); - singlePartitionDescs = partitionDesc.getSinglePartitionDescs(); - } - - private Table matchTable(Map olapTables) throws AnalysisException { - Table matched = null; - for (SlotRef column : columns) { - TableIf table = olapTables.get(column.getDesc().getParent().getTable().getName()); - if (table != null) { - if (!(table instanceof OlapTable)) { - throw new AnalysisException( - "Can not get the partition information from a table whose type isn't OLAP."); - } - if (matched != null && !matched.getName().equals(table.getName())) { - throw new AnalysisException("The partition columns must be in the same table."); - } else if (matched == null) { - matched = (Table) table; - } - } - } - if (matched == null) { - throw new AnalysisException("The partition columns doesn't match the ones in base table."); - } - PartitionInfo partitionInfo = ((OlapTable) matched).getPartitionInfo(); - List partitionColumns = partitionInfo.getPartitionColumns(); - if (!columns.stream().map(SlotRef::getColumn).collect(Collectors.toList()).equals(partitionColumns)) { - throw new AnalysisException("The partition columns doesn't match the ones in base table " - + matched.getName() + "."); - } - return matched; - } - - private List toMVPartitionColumnNames(String tableName, List partitionColNames, QueryStmt queryStmt) - throws AnalysisException { - List mvPartitionColumnNames = Lists.newArrayListWithCapacity(partitionColNames.size()); - if (queryStmt instanceof SelectStmt) { - List items = ((SelectStmt) queryStmt).getSelectList().getItems(); - for (String partitionColName : partitionColNames) { - String mvColumnName = null; - for (int i = 0; mvColumnName == null && i < items.size(); ++i) { - SelectListItem item = items.get(i); - if (item.isStar()) { - mvColumnName = partitionColName; - } else if (item.getExpr() instanceof SlotRef) { - SlotRef slotRef = (SlotRef) item.getExpr(); - if (slotRef.getTableName().getTbl().equals(tableName) && slotRef.getColumnName() - .equals(partitionColName)) { - mvColumnName = item.getAlias() == null ? partitionColName : item.getAlias(); - } - } - } - if (mvColumnName != null) { - mvPartitionColumnNames.add(mvColumnName); - } else { - throw new AnalysisException( - "Failed to map the partition column name " + partitionColName + " to mv column"); - } - } - } else { - throw new AnalysisException("Only select statement is supported."); - } - return mvPartitionColumnNames; - } - - @Override - public PartitionInfo toPartitionInfo(List schema, Map partitionNameToId, boolean isTemp) - throws DdlException, AnalysisException { - switch (type) { - case RANGE: - return new RangePartitionDesc(partitionColNames, - singlePartitionDescs.stream().map(desc -> (AllPartitionDesc) desc) - .collect(Collectors.toList())).toPartitionInfo(schema, partitionNameToId, isTemp); - case LIST: - return new ListPartitionDesc(partitionColNames, - singlePartitionDescs.stream().map(desc -> (AllPartitionDesc) desc) - .collect(Collectors.toList())).toPartitionInfo(schema, partitionNameToId, isTemp); - default: - throw new RuntimeException("Invalid partition type."); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMultiTableMaterializedViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMultiTableMaterializedViewStmt.java deleted file mode 100644 index ee00efdab0..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMultiTableMaterializedViewStmt.java +++ /dev/null @@ -1,232 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.analysis; - -import org.apache.doris.analysis.ColumnDef.DefaultValue; -import org.apache.doris.analysis.MVRefreshInfo.BuildMode; -import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.DatabaseIf; -import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.TableIf; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Config; -import org.apache.doris.common.ErrorCode; -import org.apache.doris.common.ErrorReport; -import org.apache.doris.common.UserException; -import org.apache.doris.common.util.PrintableMap; -import org.apache.doris.datasource.CatalogIf; -import org.apache.doris.datasource.InternalCatalog; - -import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; - -public class CreateMultiTableMaterializedViewStmt extends CreateTableStmt { - private final String mvName; - private final BuildMode buildMode; - private final MVRefreshInfo refreshInfo; - private final QueryStmt queryStmt; - private final Map tables = Maps.newHashMap(); - - public CreateMultiTableMaterializedViewStmt(String mvName, BuildMode buildMode, - MVRefreshInfo refreshInfo, KeysDesc keyDesc, PartitionDesc partitionDesc, DistributionDesc distributionDesc, - Map properties, QueryStmt queryStmt) { - this.mvName = mvName; - this.buildMode = buildMode; - this.refreshInfo = refreshInfo; - this.queryStmt = queryStmt; - - this.keysDesc = keyDesc; - this.partitionDesc = partitionDesc; - this.distributionDesc = distributionDesc; - this.properties = properties; - engineName = DEFAULT_ENGINE_NAME; - } - - @Override - public void analyze(Analyzer analyzer) throws UserException { - if (!Config.enable_mtmv) { - throw new UserException("Multi table materialized view was not graduated." - + " You should set `enable_mtmv = true` in fe to enabled it manually."); - } - refreshInfo.analyze(analyzer); - queryStmt.setNeedToSql(true); - queryStmt.setToSQLWithHint(true); - queryStmt.analyze(analyzer); - if (queryStmt instanceof SelectStmt) { - analyzeSelectClause((SelectStmt) queryStmt); - } - String defaultDb = analyzer.getDefaultDb(); - if (Strings.isNullOrEmpty(defaultDb)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR); - } - - tableName = new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, defaultDb, mvName); - if (partitionDesc != null) { - ((ColumnPartitionDesc) partitionDesc).analyze(analyzer, this); - } - super.analyze(analyzer); - } - - private void analyzeSelectClause(SelectStmt selectStmt) throws AnalysisException { - for (TableRef tableRef : selectStmt.getTableRefs()) { - TableIf table = null; - if (tableRef instanceof BaseTableRef) { - String dbName = tableRef.getName().getDb(); - String ctlName = tableRef.getName().getCtl(); - CatalogIf catalogIf = Env.getCurrentEnv().getCatalogMgr().getCatalog(ctlName); - if (catalogIf == null) { - throw new AnalysisException("Failed to get the catalog: " + ctlName); - } - Optional dbIf = catalogIf.getDb(dbName); - if (!dbIf.isPresent()) { - throw new AnalysisException("Failed to get the database: " + dbName); - } - Optional tableIf = dbIf.get().getTable(tableRef.getName().getTbl()); - if (!tableIf.isPresent()) { - throw new AnalysisException("Failed to get the table: " + tableRef.getName().getTbl()); - } - table = tableIf.orElse(null); - } else if (tableRef instanceof InlineViewRef) { - InlineViewRef inlineViewRef = (InlineViewRef) tableRef; - table = inlineViewRef.getDesc().getTable(); - } - if (table == null) { - throw new AnalysisException("Failed to get the table by " + tableRef); - } - tables.put(table.getName(), table); - tables.put(tableRef.getAlias(), table); - } - checkSelectListItems(selectStmt.getSelectList().getItems()); - columnDefs = generateColumnDefinitions(selectStmt); - } - - private void checkSelectListItems(List items) throws AnalysisException { - for (SelectListItem item : items) { - if (item.isStar()) { - continue; - } - Expr itemExpr = item.getExpr(); - String alias = item.getAlias(); - if (itemExpr instanceof SlotRef) { - continue; - } else if (itemExpr instanceof FunctionCallExpr && ((FunctionCallExpr) itemExpr).isAggregateFunction()) { - FunctionCallExpr functionCallExpr = (FunctionCallExpr) itemExpr; - String functionName = functionCallExpr.getFnName().getFunction(); - MVColumnPattern mvColumnPattern = CreateMaterializedViewStmt.FN_NAME_TO_PATTERN - .get(functionName.toLowerCase()); - if (mvColumnPattern == null) { - throw new AnalysisException( - "Materialized view does not support this function:" + functionCallExpr.toSqlImpl()); - } - if (!mvColumnPattern.match(functionCallExpr)) { - throw new AnalysisException( - "The function " + functionName + " must match pattern:" + mvColumnPattern); - } - if (StringUtils.isEmpty(alias)) { - throw new AnalysisException("Function expr: " + functionName + " must have a alias name for MTMV."); - } - } else { - throw new AnalysisException( - "Materialized view does not support this expr:" + itemExpr.toSqlImpl()); - } - } - } - - private List generateColumnDefinitions(SelectStmt selectStmt) throws AnalysisException { - List schema = generateSchema(selectStmt); - return schema.stream() - .map(column -> new ColumnDef( - column.getName(), - new TypeDef(column.getType()), - column.isKey(), - null, - column.isAllowNull(), - column.isAutoInc(), - new DefaultValue(column.getDefaultValue() != null, column.getDefaultValue()), - column.getComment()) - ).collect(Collectors.toList()); - } - - private List generateSchema(SelectStmt selectStmt) throws AnalysisException { - ArrayList resultExprs = selectStmt.getResultExprs(); - ArrayList colLabels = selectStmt.getColLabels(); - Map uniqueMVColumnItems = Maps.newLinkedHashMap(); - for (int i = 0; i < resultExprs.size(); i++) { - Column column = generateMTMVColumn(resultExprs.get(i), colLabels.get(i)); - if (uniqueMVColumnItems.put(column.getName(), column) != null) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DUP_FIELDNAME, column.getName()); - } - } - - return Lists.newArrayList(uniqueMVColumnItems.values()); - } - - private Column generateMTMVColumn(Expr expr, String colLabel) { - return new Column(colLabel.toLowerCase(), expr.getType(), true); - } - - @Override - public String toSql() { - StringBuilder sb = new StringBuilder(); - sb.append("CREATE MATERIALIZED VIEW ").append(mvName).append(" BUILD ").append(buildMode.toString()); - if (refreshInfo != null) { - sb.append(" ").append(refreshInfo); - } - if (partitionDesc != null) { - sb.append(" ").append(partitionDesc); - } - if (distributionDesc != null) { - sb.append(" ").append(distributionDesc); - } - if (properties != null && !properties.isEmpty()) { - sb.append("\nPROPERTIES ("); - sb.append(new PrintableMap<>(properties, " = ", true, true, true)); - sb.append(")"); - } - sb.append(" AS ").append(queryStmt.toSql()); - return sb.toString(); - } - - public String getMVName() { - return mvName; - } - - public Map getTables() { - return tables; - } - - public MVRefreshInfo getRefreshInfo() { - return refreshInfo; - } - - public QueryStmt getQueryStmt() { - return queryStmt; - } - - public BuildMode getBuildMode() { - return buildMode; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index f56de294d9..aecee1465d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -559,8 +559,7 @@ public class CreateTableStmt extends DdlStmt { tableName.getCtl(), tableName.getDb(), properties); // analyze partition if (partitionDesc != null) { - if (partitionDesc instanceof ListPartitionDesc || partitionDesc instanceof RangePartitionDesc - || partitionDesc instanceof ColumnPartitionDesc) { + if (partitionDesc instanceof ListPartitionDesc || partitionDesc instanceof RangePartitionDesc) { partitionDesc.analyze(columnDefs, properties); } else { throw new AnalysisException("Currently only support range" diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MVRefreshInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MVRefreshInfo.java deleted file mode 100644 index 93d7389dae..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MVRefreshInfo.java +++ /dev/null @@ -1,91 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.analysis; - -import org.apache.doris.common.UserException; - -import com.google.gson.annotations.SerializedName; - -public class MVRefreshInfo { - @SerializedName("neverRefresh") - private boolean neverRefresh; - @SerializedName("refreshMethod") - private RefreshMethod refreshMethod; - @SerializedName("triggerInfo") - private MVRefreshTriggerInfo triggerInfo; - - // For deserialization - public MVRefreshInfo() {} - - public MVRefreshInfo(boolean neverRefresh) { - this(neverRefresh, RefreshMethod.COMPLETE, null); - } - - public MVRefreshInfo(RefreshMethod method, MVRefreshTriggerInfo trigger) { - this(trigger == null, method, trigger); - } - - public MVRefreshInfo(boolean neverRefresh, RefreshMethod method, MVRefreshTriggerInfo trigger) { - this.neverRefresh = neverRefresh; - refreshMethod = method; - triggerInfo = trigger; - } - - void analyze(Analyzer analyzer) throws UserException { - if (triggerInfo != null) { - triggerInfo.analyze(analyzer); - } - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - if (neverRefresh) { - sb.append(" NEVER REFRESH "); - } else { - sb.append(" REFRESH "); - sb.append(refreshMethod.toString()); - sb.append(triggerInfo.toString()); - } - return sb.toString(); - } - - public boolean isNeverRefresh() { - return neverRefresh; - } - - public RefreshMethod getRefreshMethod() { - return refreshMethod; - } - - public MVRefreshTriggerInfo getTriggerInfo() { - return triggerInfo; - } - - public enum RefreshMethod { - COMPLETE, FAST, FORCE - } - - public enum BuildMode { - IMMEDIATE, DEFERRED - } - - public enum RefreshTrigger { - DEMAND, COMMIT, INTERVAL - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MVRefreshIntervalTriggerInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MVRefreshIntervalTriggerInfo.java deleted file mode 100644 index cde1faaca7..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MVRefreshIntervalTriggerInfo.java +++ /dev/null @@ -1,63 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.analysis; - -import com.google.gson.annotations.SerializedName; - -public class MVRefreshIntervalTriggerInfo { - @SerializedName("startTime") - private String startTime; - @SerializedName("interval") - private long interval; - @SerializedName("timeUnit") - private String timeUnit; - - // For deserialization - public MVRefreshIntervalTriggerInfo() { - } - - public MVRefreshIntervalTriggerInfo(String startTime, long interval, String timeUnit) { - this.startTime = startTime; - this.interval = interval; - this.timeUnit = timeUnit; - } - - public String getStartTime() { - return startTime; - } - - public long getInterval() { - return interval; - } - - public String getTimeUnit() { - return timeUnit; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - if (startTime != null) { - sb.append(" START WITH \"").append(startTime).append("\""); - } - if (interval > 0) { - sb.append(" NEXT ").append(interval).append(" ").append(timeUnit); - } - return sb.toString(); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MVRefreshTriggerInfo.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MVRefreshTriggerInfo.java deleted file mode 100644 index 8ed19d09a2..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MVRefreshTriggerInfo.java +++ /dev/null @@ -1,77 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.analysis; - -import org.apache.doris.analysis.MVRefreshInfo.RefreshTrigger; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.UserException; - -import com.google.gson.annotations.SerializedName; - -public class MVRefreshTriggerInfo { - @SerializedName("refreshTrigger") - private RefreshTrigger refreshTrigger; - @SerializedName("intervalTrigger") - private MVRefreshIntervalTriggerInfo intervalTrigger; - - // For deserialization - public MVRefreshTriggerInfo() {} - - public MVRefreshTriggerInfo(RefreshTrigger trigger) { - this(trigger, null); - } - - public MVRefreshTriggerInfo(MVRefreshIntervalTriggerInfo trigger) { - this(RefreshTrigger.INTERVAL, trigger); - } - - public MVRefreshTriggerInfo(RefreshTrigger refreshTrigger, MVRefreshIntervalTriggerInfo intervalTrigger) { - this.refreshTrigger = refreshTrigger; - this.intervalTrigger = intervalTrigger; - } - - void analyze(Analyzer analyzer) throws UserException { - if (refreshTrigger == RefreshTrigger.INTERVAL && (intervalTrigger == null || ( - intervalTrigger.getStartTime() == null && intervalTrigger.getInterval() < 0))) { - throw new AnalysisException("Start time or interval is required."); - } else if (refreshTrigger == null) { - throw new AnalysisException("refresh trigger is required."); - } - } - - - public RefreshTrigger getRefreshTrigger() { - return refreshTrigger; - } - - public MVRefreshIntervalTriggerInfo getIntervalTrigger() { - return intervalTrigger; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - if (refreshTrigger != RefreshTrigger.INTERVAL) { - sb.append(" ON "); - sb.append(refreshTrigger.toString()); - } else { - sb.append(intervalTrigger.toString()); - } - return sb.toString(); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshMaterializedViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshMaterializedViewStmt.java deleted file mode 100644 index 218e4673cf..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshMaterializedViewStmt.java +++ /dev/null @@ -1,76 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.analysis; - -import org.apache.doris.analysis.MVRefreshInfo.RefreshMethod; -import org.apache.doris.catalog.Env; -import org.apache.doris.common.ErrorCode; -import org.apache.doris.common.ErrorReport; -import org.apache.doris.common.UserException; -import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.qe.ConnectContext; - -/** - * REFRESH MATERIALIZED VIEW [db_name]. [complete|fast]; - * - * Parameters - * [complete|fast]: complete or incremental refresh. - * mv_name: The name of the materialized view to refresh. - */ -public class RefreshMaterializedViewStmt extends DdlStmt { - - private TableName mvName; - private RefreshMethod refreshMethod; - - public RefreshMaterializedViewStmt(TableName mvName, RefreshMethod refreshMethod) { - this.mvName = mvName; - if (refreshMethod == null) { - this.refreshMethod = RefreshMethod.COMPLETE; - } - this.refreshMethod = refreshMethod; - } - - public TableName getMvName() { - return mvName; - } - - public RefreshMethod getRefreshMethod() { - return refreshMethod; - } - - @Override - public void analyze(Analyzer analyzer) throws UserException { - mvName.analyze(analyzer); - - // check access - if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ConnectContext.get(), mvName.getDb(), - mvName.getTbl(), PrivPredicate.ADMIN)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); - } - } - - @Override - public String toSql() { - StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append("REFRESH MATERIALIZED VIEW ") - .append(mvName.toSql()) - .append(" ") - .append(refreshMethod.toString()); - return stringBuilder.toString(); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowMTMVJobStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowMTMVJobStmt.java deleted file mode 100644 index cdba6ad9ac..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowMTMVJobStmt.java +++ /dev/null @@ -1,143 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.analysis; - -import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.ScalarType; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.UserException; -import org.apache.doris.mtmv.metadata.MTMVJob; -import org.apache.doris.qe.ShowResultSetMetaData; - -import com.google.common.base.Strings; - -public class ShowMTMVJobStmt extends ShowStmt { - - private final String jobName; // optional - private final String dbName; // optional - private final TableName mvName; // optional - - private String analyzedDbName; - - public ShowMTMVJobStmt() { - this.jobName = null; - this.dbName = null; - this.mvName = null; - } - - public ShowMTMVJobStmt(String jobName) { - this.jobName = jobName; - this.dbName = null; - this.mvName = null; - } - - public ShowMTMVJobStmt(String dbName, TableName mvName) { - this.jobName = null; - this.dbName = dbName; - this.mvName = mvName; - } - - public boolean isShowAllJobs() { - return analyzedDbName == null && mvName == null && jobName == null; - } - - public boolean isShowAllJobsFromDb() { - return analyzedDbName != null && mvName == null; - } - - public boolean isShowAllJobsOnMv() { - return mvName != null; - } - - public boolean isSpecificJob() { - return jobName != null; - } - - public String getDbName() { - return analyzedDbName; - } - - public String getMVName() { - return mvName != null ? mvName.getTbl() : null; - } - - public String getJobName() { - return jobName; - } - - @Override - public void analyze(Analyzer analyzer) throws UserException { - super.analyze(analyzer); - if (dbName != null && mvName != null && mvName.getDb() != null && !dbName.equals(mvName.getDb())) { - throw new UserException("Database name should be same when they both been set."); - } - analyzedDbName = dbName; - if (Strings.isNullOrEmpty(analyzedDbName)) { - if (mvName != null) { - analyzedDbName = mvName.getDb(); - } - if (Strings.isNullOrEmpty(analyzedDbName)) { - analyzedDbName = analyzer.getDefaultDb(); - } - } - if (!Strings.isNullOrEmpty(analyzedDbName)) { - analyzedDbName = ClusterNamespace.getFullName(getClusterName(), analyzedDbName); - } - } - - @Override - public ShowResultSetMetaData getMetaData() { - ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); - - for (String title : MTMVJob.SHOW_TITLE_NAMES) { - if (title.equals("Query")) { - builder.addColumn(new Column(title, ScalarType.createVarchar(10240))); - } else { - builder.addColumn(new Column(title, ScalarType.createVarchar(1024))); - } - } - return builder.build(); - } - - @Override - public String toSql() { - StringBuilder sb = new StringBuilder(); - sb.append("SHOW MTMV JOB"); - - if (!Strings.isNullOrEmpty(jobName)) { - sb.append(" FOR "); - sb.append(getJobName()); - } - if (!Strings.isNullOrEmpty(dbName)) { - sb.append(" FROM "); - sb.append(ClusterNamespace.getNameFromFullName(dbName)); - } - if (mvName != null) { - sb.append(" ON "); - sb.append(mvName.toSql()); - } - - return sb.toString(); - } - - @Override - public String toString() { - return toSql(); - } - -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowMTMVTaskStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowMTMVTaskStmt.java deleted file mode 100644 index 4af8798348..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowMTMVTaskStmt.java +++ /dev/null @@ -1,142 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.analysis; - -import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.ScalarType; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.UserException; -import org.apache.doris.mtmv.metadata.MTMVTask; -import org.apache.doris.qe.ShowResultSetMetaData; - -import com.google.common.base.Strings; - -public class ShowMTMVTaskStmt extends ShowStmt { - private final String taskId; // optional - private final String dbName; // optional - private final TableName mvName; // optional - - private String analyzedDbName; - - public ShowMTMVTaskStmt() { - this.taskId = null; - this.dbName = null; - this.mvName = null; - } - - public ShowMTMVTaskStmt(String taskId) { - this.taskId = taskId; - this.dbName = null; - this.mvName = null; - } - - public ShowMTMVTaskStmt(String dbName, TableName mvName) { - this.taskId = null; - this.dbName = dbName; - this.mvName = mvName; - } - - public boolean isShowAllTasks() { - return analyzedDbName == null && mvName == null && taskId == null; - } - - public boolean isShowAllTasksFromDb() { - return analyzedDbName != null && mvName == null; - } - - public boolean isShowAllTasksOnMv() { - return mvName != null; - } - - public boolean isSpecificTask() { - return taskId != null; - } - - public String getDbName() { - return analyzedDbName; - } - - public String getMVName() { - return mvName != null ? mvName.getTbl() : null; - } - - public String getTaskId() { - return taskId; - } - - @Override - public void analyze(Analyzer analyzer) throws UserException { - super.analyze(analyzer); - if (dbName != null && mvName != null && mvName.getDb() != null && !dbName.equals(mvName.getDb())) { - throw new UserException("Database name should be same when they both been set."); - } - analyzedDbName = dbName; - if (Strings.isNullOrEmpty(analyzedDbName)) { - if (mvName != null) { - analyzedDbName = mvName.getDb(); - } - if (Strings.isNullOrEmpty(analyzedDbName)) { - analyzedDbName = analyzer.getDefaultDb(); - } - } - if (!Strings.isNullOrEmpty(analyzedDbName)) { - analyzedDbName = ClusterNamespace.getFullName(getClusterName(), analyzedDbName); - } - } - - @Override - public ShowResultSetMetaData getMetaData() { - ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); - - for (String title : MTMVTask.SHOW_TITLE_NAMES) { - if (title.equals("Query") || title.equals("ErrorMessage")) { - builder.addColumn(new Column(title, ScalarType.createVarchar(10240))); - } else { - builder.addColumn(new Column(title, ScalarType.createVarchar(1024))); - } - } - return builder.build(); - } - - - @Override - public String toSql() { - StringBuilder sb = new StringBuilder(); - sb.append("SHOW MTMV TASK"); - - if (taskId != null) { - sb.append(" FOR "); - sb.append(getTaskId()); - } - if (dbName != null) { - sb.append(" FROM "); - sb.append(ClusterNamespace.getNameFromFullName(dbName)); - } - if (mvName != null) { - sb.append(" ON "); - sb.append(mvName.toSql()); - } - - return sb.toString(); - } - - @Override - public String toString() { - return toSql(); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index caa7d50d91..9383c8b926 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -38,7 +38,6 @@ import org.apache.doris.analysis.AlterDatabasePropertyStmt; import org.apache.doris.analysis.AlterDatabaseQuotaStmt; import org.apache.doris.analysis.AlterDatabaseQuotaStmt.QuotaType; import org.apache.doris.analysis.AlterDatabaseRename; -import org.apache.doris.analysis.AlterMaterializedViewStmt; import org.apache.doris.analysis.AlterSystemStmt; import org.apache.doris.analysis.AlterTableStmt; import org.apache.doris.analysis.AlterViewStmt; @@ -50,7 +49,6 @@ import org.apache.doris.analysis.ColumnRenameClause; import org.apache.doris.analysis.CreateDbStmt; import org.apache.doris.analysis.CreateFunctionStmt; import org.apache.doris.analysis.CreateMaterializedViewStmt; -import org.apache.doris.analysis.CreateMultiTableMaterializedViewStmt; import org.apache.doris.analysis.CreateTableAsSelectStmt; import org.apache.doris.analysis.CreateTableLikeStmt; import org.apache.doris.analysis.CreateTableStmt; @@ -70,7 +68,6 @@ import org.apache.doris.analysis.PartitionRenameClause; import org.apache.doris.analysis.RecoverDbStmt; import org.apache.doris.analysis.RecoverPartitionStmt; import org.apache.doris.analysis.RecoverTableStmt; -import org.apache.doris.analysis.RefreshMaterializedViewStmt; import org.apache.doris.analysis.ReplacePartitionClause; import org.apache.doris.analysis.RestoreStmt; import org.apache.doris.analysis.RollupRenameClause; @@ -175,11 +172,9 @@ import org.apache.doris.master.MetaHelper; import org.apache.doris.master.PartitionInMemoryInfoCollector; import org.apache.doris.meta.MetaContext; import org.apache.doris.metric.MetricRepo; -import org.apache.doris.mtmv.MTMVJobManager; import org.apache.doris.mysql.privilege.AccessControllerManager; import org.apache.doris.mysql.privilege.Auth; import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.persist.AlterMultiMaterializedView; import org.apache.doris.persist.AutoIncrementIdUpdateLog; import org.apache.doris.persist.BackendReplicasInfo; import org.apache.doris.persist.BackendTabletsInfo; @@ -473,8 +468,6 @@ public class Env { private PolicyMgr policyMgr; - private MTMVJobManager mtmvJobManager; - private AnalysisManager analysisManager; private ExternalMetaCacheMgr extMetaCacheMgr; @@ -580,10 +573,6 @@ public class Env { return catalogMgr; } - public MTMVJobManager getMTMVJobManager() { - return mtmvJobManager; - } - public ExternalMetaCacheMgr getExtMetaCacheMgr() { return extMetaCacheMgr; } @@ -725,7 +714,6 @@ public class Env { this.auditEventProcessor = new AuditEventProcessor(this.pluginMgr); this.refreshManager = new RefreshManager(); this.policyMgr = new PolicyMgr(); - this.mtmvJobManager = new MTMVJobManager(); this.extMetaCacheMgr = new ExternalMetaCacheMgr(); this.analysisManager = new AnalysisManager(); this.statisticsCleaner = new StatisticsCleaner(); @@ -1583,8 +1571,6 @@ public class Env { if (Config.enable_hms_events_incremental_sync) { metastoreEventsProcessor.start(); } - // start mtmv jobManager - mtmvJobManager.start(); getRefreshManager().start(); // binlog gcer @@ -1635,8 +1621,6 @@ public class Env { MetricRepo.init(); - // stop mtmv scheduler - mtmvJobManager.stop(); if (analysisManager != null) { analysisManager.getStatisticsCache().preHeat(); } @@ -2102,15 +2086,6 @@ public class Env { return checksum; } - /** - * Load mtmv jobManager. - **/ - public long loadMTMVJobManager(DataInputStream in, long checksum) throws IOException { - this.mtmvJobManager = MTMVJobManager.read(in, checksum); - LOG.info("finished replay mtmv job and tasks from image"); - return checksum; - } - /** * Load global function. **/ @@ -2362,12 +2337,6 @@ public class Env { return checksum; } - public long saveMTMVJobManager(CountingDataOutputStream out, long checksum) throws IOException { - Env.getCurrentEnv().getMTMVJobManager().write(out, checksum); - LOG.info("Save mtmv job and tasks to image"); - return checksum; - } - public long saveGlobalFunction(CountingDataOutputStream out, long checksum) throws IOException { this.globalFunctionMgr.write(out); LOG.info("Save global function to image"); @@ -3032,10 +3001,6 @@ public class Env { } sb.append("\n) ENGINE="); sb.append(table.getType().name()); - } else { - MaterializedView materializedView = ((MaterializedView) table); - sb.append("\n").append("BUILD ").append(materializedView.getBuildMode()) - .append(materializedView.getRefreshInfo().toString()); } if (table.getType() == TableType.OLAP || table.getType() == TableType.MATERIALIZED_VIEW) { @@ -3434,10 +3399,6 @@ public class Env { sb.append("\n)"); } - if (table.getType() == TableType.MATERIALIZED_VIEW) { - sb.append("\nAS ").append(((MaterializedView) table).getQuery()); - } - createTableStmt.add(sb + ";"); // 2. add partition @@ -4060,11 +4021,6 @@ public class Env { this.alter.processAlterTable(stmt); } - public void alterMaterializedView(AlterMaterializedViewStmt stmt) throws UserException { - AlterMultiMaterializedView alter = new AlterMultiMaterializedView(stmt.getTable(), stmt.getRefreshInfo()); - this.alter.processAlterMaterializedView(alter, false); - } - /** * used for handling AlterViewStmt (the ALTER VIEW command). */ @@ -4077,18 +4033,10 @@ public class Env { this.alter.processCreateMaterializedView(stmt); } - public void createMultiTableMaterializedView(CreateMultiTableMaterializedViewStmt stmt) throws UserException { - this.alter.processCreateMultiTableMaterializedView(stmt); - } - public void dropMaterializedView(DropMaterializedViewStmt stmt) throws DdlException, MetaNotFoundException { this.alter.processDropMaterializedView(stmt); } - public void refreshMaterializedView(RefreshMaterializedViewStmt stmt) throws DdlException, MetaNotFoundException { - this.alter.processRefreshMaterializedView(stmt); - } - /* * used for handling CancelAlterStmt (for client is the CANCEL ALTER * command). including SchemaChangeHandler and RollupHandler diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedView.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedView.java deleted file mode 100644 index 3de69dda8a..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedView.java +++ /dev/null @@ -1,129 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.catalog; - -import org.apache.doris.analysis.MVRefreshInfo; -import org.apache.doris.analysis.MVRefreshInfo.BuildMode; -import org.apache.doris.catalog.OlapTableFactory.MaterializedViewParams; -import org.apache.doris.common.FeConstants; -import org.apache.doris.common.io.Text; -import org.apache.doris.meta.MetaContext; -import org.apache.doris.persist.gson.GsonUtils; - -import com.google.gson.annotations.SerializedName; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInput; -import java.io.DataInputStream; -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.IOException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; - - -public class MaterializedView extends OlapTable { - @SerializedName("buildMode") - private BuildMode buildMode; - @SerializedName("refreshInfo") - private MVRefreshInfo refreshInfo; - @SerializedName("query") - private String query; - - private final ReentrantLock mvTaskLock = new ReentrantLock(true); - - public boolean tryLockMVTask() { - try { - return mvTaskLock.tryLock(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - return false; - } - } - - public void unLockMVTask() { - this.mvTaskLock.unlock(); - } - - // For deserialization - public MaterializedView() { - type = TableType.MATERIALIZED_VIEW; - } - - MaterializedView(MaterializedViewParams params) { - super( - params.tableId, - params.tableName, - params.schema, - params.keysType, - params.partitionInfo, - params.distributionInfo - ); - type = TableType.MATERIALIZED_VIEW; - buildMode = params.buildMode; - refreshInfo = params.mvRefreshInfo; - query = params.queryStmt.toSqlWithHint(); - } - - public BuildMode getBuildMode() { - return buildMode; - } - - public MVRefreshInfo getRefreshInfo() { - return refreshInfo; - } - - public void setRefreshInfo(MVRefreshInfo info) { - refreshInfo = info; - } - - public String getQuery() { - return query; - } - - @Override - public void write(DataOutput out) throws IOException { - super.write(out); - Text.writeString(out, GsonUtils.GSON.toJson(this)); - } - - @Override - public void readFields(DataInput in) throws IOException { - super.readFields(in); - MaterializedView materializedView = GsonUtils.GSON.fromJson(Text.readString(in), this.getClass()); - refreshInfo = materializedView.refreshInfo; - query = materializedView.query; - buildMode = materializedView.buildMode; - } - - public MaterializedView clone(String mvName) throws IOException { - MetaContext metaContext = new MetaContext(); - metaContext.setMetaVersion(FeConstants.meta_version); - metaContext.setThreadLocalInfo(); - try { - ByteArrayOutputStream out = new ByteArrayOutputStream(256); - MaterializedView cloned = new MaterializedView(); - this.write(new DataOutputStream(out)); - cloned.readFields(new DataInputStream(new ByteArrayInputStream(out.toByteArray()))); - cloned.setName(mvName); - return cloned; - } finally { - MetaContext.remove(); - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTableFactory.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTableFactory.java index 3e0a1eba5c..b17b58abb5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTableFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTableFactory.java @@ -17,12 +17,8 @@ package org.apache.doris.catalog; -import org.apache.doris.analysis.CreateMultiTableMaterializedViewStmt; import org.apache.doris.analysis.CreateTableStmt; import org.apache.doris.analysis.DdlStmt; -import org.apache.doris.analysis.MVRefreshInfo; -import org.apache.doris.analysis.MVRefreshInfo.BuildMode; -import org.apache.doris.analysis.QueryStmt; import org.apache.doris.catalog.TableIf.TableType; import com.google.common.base.Preconditions; @@ -44,18 +40,10 @@ public class OlapTableFactory { public TableIndexes indexes; } - public static class MaterializedViewParams extends BuildParams { - public MVRefreshInfo.BuildMode buildMode; - public MVRefreshInfo mvRefreshInfo; - public QueryStmt queryStmt; - } - private BuildParams params; public static TableType getTableType(DdlStmt stmt) { - if (stmt instanceof CreateMultiTableMaterializedViewStmt) { - return TableType.MATERIALIZED_VIEW; - } else if (stmt instanceof CreateTableStmt) { + if (stmt instanceof CreateTableStmt) { return TableType.OLAP; } else { throw new IllegalArgumentException("Invalid DDL statement: " + stmt.toSql()); @@ -63,32 +51,22 @@ public class OlapTableFactory { } public OlapTableFactory init(TableType type) { - params = (type == TableType.OLAP) ? new OlapTableParams() : new MaterializedViewParams(); + params = new OlapTableParams(); return this; } public Table build() { Preconditions.checkNotNull(params, "The factory isn't initialized."); - - if (params instanceof OlapTableParams) { - OlapTableParams olapTableParams = (OlapTableParams) params; - return new OlapTable( - olapTableParams.tableId, - olapTableParams.tableName, - olapTableParams.schema, - olapTableParams.keysType, - olapTableParams.partitionInfo, - olapTableParams.distributionInfo, - olapTableParams.indexes - ); - } else { - MaterializedViewParams materializedViewParams = (MaterializedViewParams) params; - return new MaterializedView(materializedViewParams); - } - } - - public BuildParams getBuildParams() { - return params; + OlapTableParams olapTableParams = (OlapTableParams) params; + return new OlapTable( + olapTableParams.tableId, + olapTableParams.tableName, + olapTableParams.schema, + olapTableParams.keysType, + olapTableParams.partitionInfo, + olapTableParams.distributionInfo, + olapTableParams.indexes + ); } public OlapTableFactory withTableId(long tableId) { @@ -129,38 +107,8 @@ public class OlapTableFactory { return this; } - public OlapTableFactory withQueryStmt(QueryStmt queryStmt) { - Preconditions.checkState(params instanceof MaterializedViewParams, "Invalid argument for " - + params.getClass().getSimpleName()); - MaterializedViewParams materializedViewParams = (MaterializedViewParams) params; - materializedViewParams.queryStmt = queryStmt; - return this; - } - - private OlapTableFactory withBuildMode(BuildMode buildMode) { - MaterializedViewParams materializedViewParams = (MaterializedViewParams) params; - materializedViewParams.buildMode = buildMode; - return this; - } - - public OlapTableFactory withRefreshInfo(MVRefreshInfo mvRefreshInfo) { - Preconditions.checkState(params instanceof MaterializedViewParams, "Invalid argument for " - + params.getClass().getSimpleName()); - MaterializedViewParams materializedViewParams = (MaterializedViewParams) params; - materializedViewParams.mvRefreshInfo = mvRefreshInfo; - return this; - } - public OlapTableFactory withExtraParams(DdlStmt stmt) { - boolean isMaterializedView = stmt instanceof CreateMultiTableMaterializedViewStmt; - if (!isMaterializedView) { - CreateTableStmt createOlapTableStmt = (CreateTableStmt) stmt; - return withIndexes(new TableIndexes(createOlapTableStmt.getIndexes())); - } else { - CreateMultiTableMaterializedViewStmt createMVStmt = (CreateMultiTableMaterializedViewStmt) stmt; - return withBuildMode(createMVStmt.getBuildMode()) - .withRefreshInfo(createMVStmt.getRefreshInfo()) - .withQueryStmt(createMVStmt.getQueryStmt()); - } + CreateTableStmt createOlapTableStmt = (CreateTableStmt) stmt; + return withIndexes(new TableIndexes(createOlapTableStmt.getIndexes())); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java index cdd179a347..65d8cc89c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java @@ -363,8 +363,6 @@ public abstract class Table extends MetaObject implements Writable, TableIf { TableType type = TableType.valueOf(Text.readString(in)); if (type == TableType.OLAP) { table = new OlapTable(); - } else if (type == TableType.MATERIALIZED_VIEW) { - table = new MaterializedView(); } else if (type == TableType.ODBC) { table = new OdbcTable(); } else if (type == TableType.MYSQL) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 8ecdf643e8..9b8985a8c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -79,7 +79,6 @@ import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.MaterializedIndex.IndexState; import org.apache.doris.catalog.MaterializedIndexMeta; -import org.apache.doris.catalog.MaterializedView; import org.apache.doris.catalog.MetaIdGenerator.IdGeneratorBuffer; import org.apache.doris.catalog.MysqlCompatibleDatabase; import org.apache.doris.catalog.MysqlDb; @@ -137,8 +136,6 @@ import org.apache.doris.datasource.property.constants.HMSProperties; import org.apache.doris.external.elasticsearch.EsRepository; import org.apache.doris.external.iceberg.IcebergCatalogMgr; import org.apache.doris.external.iceberg.IcebergTableCreationRecordMgr; -import org.apache.doris.mtmv.MTMVJobFactory; -import org.apache.doris.mtmv.metadata.MTMVJob; import org.apache.doris.persist.AlterDatabasePropertyInfo; import org.apache.doris.persist.AutoIncrementIdUpdateLog; import org.apache.doris.persist.ColocatePersistInfo; @@ -888,7 +885,7 @@ public class InternalCatalog implements CatalogIf { ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "VIEW"); } } else { - if (table instanceof View || (!stmt.isMaterializedView() && table instanceof MaterializedView)) { + if (table instanceof View) { ErrorReport.reportDdlException(ErrorCode.ERR_WRONG_OBJECT, dbName, tableName, "TABLE"); } } @@ -953,13 +950,6 @@ public class InternalCatalog implements CatalogIf { Env.getCurrentEnv().onEraseOlapTable((OlapTable) table, isReplay); } } - - if (table instanceof MaterializedView) { - List dropIds = Env.getCurrentEnv().getMTMVJobManager().showJobs(db.getFullName(), table.getName()) - .stream().map(MTMVJob::getId).collect(Collectors.toList()); - Env.getCurrentEnv().getMTMVJobManager().dropJobs(dropIds, isReplay); - LOG.info("Drop related {} mv job.", dropIds.size()); - } LOG.info("finished dropping table[{}] in db[{}]", table.getName(), db.getFullName()); return true; } @@ -2601,14 +2591,6 @@ public class InternalCatalog implements CatalogIf { throw e; } - - if (olapTable instanceof MaterializedView && MTMVJobFactory.isGenerateJob((MaterializedView) olapTable)) { - List jobs = MTMVJobFactory.buildJob((MaterializedView) olapTable, db.getFullName()); - for (MTMVJob job : jobs) { - Env.getCurrentEnv().getMTMVJobManager().createJob(job, false); - } - LOG.info("Create related {} mv job.", jobs.size()); - } } private void createMysqlTable(Database db, CreateTableStmt stmt) throws DdlException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java index d47c9f6619..ba2fe4becf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java @@ -55,15 +55,9 @@ import org.apache.doris.load.loadv2.LoadJob.LoadJobStateUpdateInfo; import org.apache.doris.load.loadv2.LoadJobFinalOperation; import org.apache.doris.load.routineload.RoutineLoadJob; import org.apache.doris.load.sync.SyncJob; -import org.apache.doris.mtmv.metadata.ChangeMTMVJob; -import org.apache.doris.mtmv.metadata.DropMTMVJob; -import org.apache.doris.mtmv.metadata.DropMTMVTask; -import org.apache.doris.mtmv.metadata.MTMVJob; -import org.apache.doris.mtmv.metadata.MTMVTask; import org.apache.doris.mysql.privilege.UserPropertyInfo; import org.apache.doris.persist.AlterDatabasePropertyInfo; import org.apache.doris.persist.AlterLightSchemaChangeInfo; -import org.apache.doris.persist.AlterMultiMaterializedView; import org.apache.doris.persist.AlterRoutineLoadJobOperationLog; import org.apache.doris.persist.AlterUserOperationLog; import org.apache.doris.persist.AlterViewInfo; @@ -789,38 +783,13 @@ public class JournalEntity implements Writable { isRead = true; break; } - case OperationType.OP_CREATE_MTMV_JOB: { - data = MTMVJob.read(in); - isRead = true; - break; - } - case OperationType.OP_DROP_MTMV_JOB: { - data = DropMTMVJob.read(in); - isRead = true; - break; - } - case OperationType.OP_CHANGE_MTMV_JOB: { - data = ChangeMTMVJob.read(in); - isRead = true; - break; - } - case OperationType.OP_CREATE_MTMV_TASK: { - data = MTMVTask.read(in); - isRead = true; - break; - } - case OperationType.OP_DROP_MTMV_TASK: { - data = DropMTMVTask.read(in); - isRead = true; - break; - } - case OperationType.OP_CHANGE_MTMV_TASK: { - Text.readString(in); - isRead = true; - break; - } + case OperationType.OP_CREATE_MTMV_JOB: + case OperationType.OP_CHANGE_MTMV_JOB: + case OperationType.OP_DROP_MTMV_JOB: + case OperationType.OP_CREATE_MTMV_TASK: + case OperationType.OP_CHANGE_MTMV_TASK: + case OperationType.OP_DROP_MTMV_TASK: case OperationType.OP_ALTER_MTMV_STMT: { - data = AlterMultiMaterializedView.read(in); isRead = true; break; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobFactory.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobFactory.java deleted file mode 100644 index 3ac142d063..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobFactory.java +++ /dev/null @@ -1,117 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.mtmv; - -import org.apache.doris.analysis.MVRefreshInfo.BuildMode; -import org.apache.doris.analysis.MVRefreshInfo.RefreshMethod; -import org.apache.doris.analysis.MVRefreshInfo.RefreshTrigger; -import org.apache.doris.analysis.MVRefreshIntervalTriggerInfo; -import org.apache.doris.analysis.MVRefreshTriggerInfo; -import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.MaterializedView; -import org.apache.doris.common.Config; -import org.apache.doris.common.FeConstants; -import org.apache.doris.common.util.TimeUtils; -import org.apache.doris.mtmv.MTMVUtils.TriggerMode; -import org.apache.doris.mtmv.metadata.MTMVJob; -import org.apache.doris.mtmv.metadata.MTMVJob.JobSchedule; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.time.LocalDateTime; -import java.time.format.DateTimeParseException; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - -public class MTMVJobFactory { - private static final Logger LOG = LogManager.getLogger(MTMVTaskProcessor.class); - - public static boolean isGenerateJob(MaterializedView materializedView) { - boolean completeRefresh = materializedView.getRefreshInfo().getRefreshMethod() == RefreshMethod.COMPLETE; - BuildMode buildMode = materializedView.getBuildMode(); - MVRefreshTriggerInfo triggerInfo = materializedView.getRefreshInfo().getTriggerInfo(); - //can not generate a job when creating a temp materialized view. - if (materializedView.getName().startsWith(FeConstants.TEMP_MATERIZLIZE_DVIEW_PREFIX)) { - return false; - } - if (buildMode == BuildMode.IMMEDIATE) { - return completeRefresh; - } else { - return completeRefresh && triggerInfo != null && triggerInfo.getRefreshTrigger() == RefreshTrigger.INTERVAL; - } - } - - public static List buildJob(MaterializedView materializedView, String dbName) { - List jobs = new ArrayList<>(); - MVRefreshTriggerInfo triggerInfo = materializedView.getRefreshInfo().getTriggerInfo(); - boolean isRunPeriodJobImmediate = false; - if (triggerInfo != null && triggerInfo.getRefreshTrigger() == RefreshTrigger.INTERVAL) { - MTMVJob job = genPeriodicalJob(materializedView, dbName); - isRunPeriodJobImmediate = MTMVUtils.getDelaySeconds(job) == 0; - jobs.add(job); - } - - // if the PeriodicalJob run immediate since an early start time, don't run the immediate build. - if (!isRunPeriodJobImmediate && materializedView.getBuildMode() == BuildMode.IMMEDIATE) { - jobs.add(genOnceJob(materializedView, dbName)); - } - return jobs; - } - - private static MTMVJob genPeriodicalJob(MaterializedView materializedView, String dbName) { - String uid = UUID.randomUUID().toString(); - MTMVJob job = new MTMVJob(materializedView.getName() + "_" + uid); - job.setId(Env.getCurrentEnv().getNextId()); - job.setTriggerMode(TriggerMode.PERIODICAL); - job.setSchedule(genJobSchedule(materializedView)); - job.setDBName(dbName); - job.setMVName(materializedView.getName()); - job.setQuery(materializedView.getQuery()); - job.setCreateTime(MTMVUtils.getNowTimeStamp()); - return job; - } - - public static MTMVJob genOnceJob(MaterializedView materializedView, String dbName) { - String uid = UUID.randomUUID().toString(); - MTMVJob job = new MTMVJob(materializedView.getName() + "_" + uid); - job.setId(Env.getCurrentEnv().getNextId()); - job.setTriggerMode(TriggerMode.ONCE); - job.setDBName(dbName); - job.setMVName(materializedView.getName()); - job.setQuery(materializedView.getQuery()); - long nowTimeStamp = MTMVUtils.getNowTimeStamp(); - job.setCreateTime(nowTimeStamp); - job.setExpireTime(nowTimeStamp + Config.scheduler_mtmv_job_expired); - return job; - } - - private static JobSchedule genJobSchedule(MaterializedView materializedView) { - MVRefreshIntervalTriggerInfo info = materializedView.getRefreshInfo().getTriggerInfo().getIntervalTrigger(); - long startTime; - try { - LocalDateTime dateTime = LocalDateTime.parse(info.getStartTime(), TimeUtils.DATETIME_FORMAT); - startTime = dateTime.toEpochSecond(TimeUtils.TIME_ZONE.getRules().getOffset(dateTime)); - } catch (DateTimeParseException e) { - throw new RuntimeException(e); - } - - return new JobSchedule(startTime, info.getInterval(), MTMVUtils.getTimeUint(info.getTimeUnit())); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java deleted file mode 100644 index b58f26b863..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java +++ /dev/null @@ -1,422 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.mtmv; - -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.MaterializedView; -import org.apache.doris.catalog.TableIf.TableType; -import org.apache.doris.common.CustomThreadFactory; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.MetaNotFoundException; -import org.apache.doris.common.io.Text; -import org.apache.doris.metric.GaugeMetric; -import org.apache.doris.metric.Metric; -import org.apache.doris.metric.MetricLabel; -import org.apache.doris.metric.MetricRepo; -import org.apache.doris.mtmv.MTMVUtils.JobState; -import org.apache.doris.mtmv.metadata.ChangeMTMVJob; -import org.apache.doris.mtmv.metadata.MTMVCheckpointData; -import org.apache.doris.mtmv.metadata.MTMVJob; -import org.apache.doris.mtmv.metadata.MTMVTask; -import org.apache.doris.persist.gson.GsonUtils; - -import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.Collectors; - -public class MTMVJobManager { - private static final Logger LOG = LogManager.getLogger(MTMVJobManager.class); - - // make sure that metrics were registered only once. - private static volatile boolean metricsRegistered = false; - - private final Map idToJobMap; - private final Map nameToJobMap; - - private final MTMVTaskManager taskManager; - - private ScheduledExecutorService periodScheduler = Executors.newScheduledThreadPool(1, - new CustomThreadFactory("mtmv-job-period-scheduler")); - - private ScheduledExecutorService cleanerScheduler = Executors.newScheduledThreadPool(1, - new CustomThreadFactory("mtmv-job-cleaner-scheduler")); - - private final ReentrantReadWriteLock rwLock; - - private final AtomicBoolean isStarted = new AtomicBoolean(false); - - public MTMVJobManager() { - idToJobMap = Maps.newConcurrentMap(); - nameToJobMap = Maps.newConcurrentMap(); - rwLock = new ReentrantReadWriteLock(true); - taskManager = new MTMVTaskManager(); - } - - public void start() { - if (isStarted.compareAndSet(false, true)) { - // check the scheduler before using it - // since it may be shutdown when master change to follower without process shutdown. - if (periodScheduler.isShutdown()) { - periodScheduler = Executors.newScheduledThreadPool(1, - new CustomThreadFactory("mtmv-job-period-scheduler")); - } - - registerJobs(); - - if (cleanerScheduler.isShutdown()) { - cleanerScheduler = Executors.newScheduledThreadPool(1, - new CustomThreadFactory("mtmv-job-cleaner-scheduler")); - } - cleanerScheduler.scheduleAtFixedRate(() -> { - if (!Env.getCurrentEnv().isMaster()) { - LOG.warn("only master can run MTMVJob"); - return; - } - removeExpiredJobs(); - taskManager.removeExpiredTasks(); - }, 0, 1, TimeUnit.MINUTES); - - taskManager.startTaskScheduler(); - initMetrics(); - } - } - - private void initMetrics() { - if (metricsRegistered) { - return; - } - metricsRegistered = true; - - // total jobs - GaugeMetric totalJob = new GaugeMetric("mtmv_job", - Metric.MetricUnit.NOUNIT, "Total job number of mtmv.") { - @Override - public Integer getValue() { - return nameToJobMap.size(); - } - }; - totalJob.addLabel(new MetricLabel("type", "TOTAL-JOB")); - MetricRepo.DORIS_METRIC_REGISTER.addMetrics(totalJob); - - // active jobs - GaugeMetric activeJob = new GaugeMetric("mtmv_job", - Metric.MetricUnit.NOUNIT, "Active job number of mtmv.") { - @Override - public Integer getValue() { - int result = 0; - for (MTMVJob job : getAllJobsWithLock()) { - if (job.getState() == JobState.ACTIVE) { - result++; - } - } - return result; - } - }; - activeJob.addLabel(new MetricLabel("type", "ACTIVE-JOB")); - MetricRepo.DORIS_METRIC_REGISTER.addMetrics(activeJob); - - // total tasks - GaugeMetric totalTask = new GaugeMetric("mtmv_task", - Metric.MetricUnit.NOUNIT, "Total task number of mtmv.") { - @Override - public Integer getValue() { - return getTaskManager().getHistoryTasks().size(); - } - }; - totalTask.addLabel(new MetricLabel("type", "TOTAL-TASK")); - MetricRepo.DORIS_METRIC_REGISTER.addMetrics(totalTask); - - // running tasks - GaugeMetric runningTask = new GaugeMetric("mtmv_task", - Metric.MetricUnit.NOUNIT, "Running task number of mtmv.") { - @Override - public Integer getValue() { - return getTaskManager().getRunningTaskMap().size(); - } - }; - runningTask.addLabel(new MetricLabel("type", "RUNNING-TASK")); - MetricRepo.DORIS_METRIC_REGISTER.addMetrics(runningTask); - - // pending tasks - GaugeMetric pendingTask = new GaugeMetric("mtmv_task", - Metric.MetricUnit.NOUNIT, "Pending task number of mtmv.") { - @Override - public Integer getValue() { - return getTaskManager().getPendingTaskMap().size(); - } - }; - pendingTask.addLabel(new MetricLabel("type", "PENDING-TASK")); - MetricRepo.DORIS_METRIC_REGISTER.addMetrics(pendingTask); - - // failed tasks - GaugeMetric failedTask = new GaugeMetric("mtmv_task", - Metric.MetricUnit.NOUNIT, "Failed task number of mtmv.") { - @Override - public Integer getValue() { - return getTaskManager().getFailedTaskCount(); - } - }; - failedTask.addLabel(new MetricLabel("type", "FAILED-TASK")); - MetricRepo.DORIS_METRIC_REGISTER.addMetrics(failedTask); - } - - public void stop() { - if (isStarted.compareAndSet(true, false)) { - periodScheduler.shutdown(); - cleanerScheduler.shutdown(); - taskManager.stopTaskScheduler(); - } - } - - private void registerJobs() { - for (MTMVJob job : getAllJobsWithLock()) { - job.start(); - } - } - - public void createJob(MTMVJob job, boolean isReplay) throws DdlException { - createJobWithLock(job, isReplay); - if (!isReplay) { - job.start(); - } - } - - private void createJobWithLock(MTMVJob job, boolean isReplay) throws DdlException { - writeLock(); - try { - if (nameToJobMap.containsKey(job.getName())) { - throw new DdlException("Job [" + job.getName() + "] already exists"); - } - nameToJobMap.put(job.getName(), job); - idToJobMap.put(job.getId(), job); - if (!isReplay) { - // log job before submit any task. - Env.getCurrentEnv().getEditLog().logCreateMTMVJob(job); - } - } finally { - writeUnlock(); - } - } - - public void refreshMTMV(String dbName, String mvName) - throws DdlException, MetaNotFoundException { - Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbName); - MaterializedView mv = (MaterializedView) db.getTableOrMetaException(mvName, TableType.MATERIALIZED_VIEW); - MTMVJob mtmvJob = MTMVJobFactory.genOnceJob(mv, dbName); - createJob(mtmvJob, false); - } - - public void dropJobByName(String dbName, String mvName, boolean isReplay) { - List jobIds = Lists.newArrayList(); - for (String jobName : nameToJobMap.keySet()) { - MTMVJob job = nameToJobMap.get(jobName); - if (job.getMVName().equals(mvName) && job.getDBName().equals(dbName)) { - jobIds.add(job.getId()); - } - } - dropJobs(jobIds, isReplay); - } - - public void dropJobs(List jobIds, boolean isReplay) { - if (jobIds.isEmpty()) { - return; - } - - for (long jobId : jobIds) { - dropJob(jobId, isReplay); - } - - LOG.info("drop jobs:{}", jobIds); - } - - private void dropJob(long jobId, boolean isReplay) { - MTMVJob job = dropJobWithLock(jobId, isReplay); - if (!isReplay && job != null) { - job.stop(); - } - } - - private MTMVJob dropJobWithLock(long jobId, boolean isReplay) { - writeLock(); - try { - MTMVJob job = idToJobMap.get(jobId); - if (job == null) { - LOG.warn("drop jobId {} failed because job is null", jobId); - return null; - } - idToJobMap.remove(job.getId()); - nameToJobMap.remove(job.getName()); - if (!isReplay) { - Env.getCurrentEnv().getEditLog().logDropMTMVJob(Collections.singletonList(jobId)); - } - return job; - } finally { - writeUnlock(); - } - } - - public List showAllJobs() { - return showJobs(null); - } - - public List showJobs(String dbName) { - List jobList = Lists.newArrayList(); - if (Strings.isNullOrEmpty(dbName)) { - jobList.addAll(getAllJobsWithLock()); - } else { - jobList.addAll(getAllJobsWithLock().stream().filter(u -> u.getDBName().equals(dbName)) - .collect(Collectors.toList())); - } - return jobList.stream().sorted().collect(Collectors.toList()); - } - - public List showJobs(String dbName, String mvName) { - return showJobs(dbName).stream().filter(u -> u.getMVName().equals(mvName)).collect(Collectors.toList()); - } - - public void replayCreateJob(MTMVJob job) { - if (job.getExpireTime() > 0 && MTMVUtils.getNowTimeStamp() > job.getExpireTime()) { - return; - } - try { - createJob(job, true); - } catch (DdlException e) { - LOG.warn("failed to replay create job [{}]", job.getName(), e); - } - } - - public void replayDropJobs(List jobIds) { - dropJobs(jobIds, true); - } - - public void replayUpdateJob(ChangeMTMVJob changeJob) { - MTMVJob mtmvJob = idToJobMap.get(changeJob.getJobId()); - if (mtmvJob != null) { - mtmvJob.updateJob(changeJob, true); - } - } - - public void replayCreateJobTask(MTMVTask task) { - taskManager.replayCreateJobTask(task); - } - - public void replayDropJobTasks(List taskIds) { - taskManager.dropHistoryTasks(taskIds, true); - } - - private void removeExpiredJobs() { - long currentTimeSeconds = MTMVUtils.getNowTimeStamp(); - List jobIdsToDelete = Lists.newArrayList(); - List jobs = getAllJobsWithLock(); - for (MTMVJob job : jobs) { - // active job should not clean - if (job.getState() != MTMVUtils.JobState.COMPLETE) { - continue; - } - long expireTime = job.getExpireTime(); - if (expireTime > 0 && currentTimeSeconds > expireTime) { - jobIdsToDelete.add(job.getId()); - } - } - dropJobs(jobIdsToDelete, false); - } - - public MTMVJob getJob(String jobName) { - return nameToJobMap.get(jobName); - } - - private List getAllJobsWithLock() { - readLock(); - try { - return Lists.newArrayList(nameToJobMap.values()); - } finally { - readUnlock(); - } - - } - - public MTMVTaskManager getTaskManager() { - return taskManager; - } - - public ScheduledExecutorService getPeriodScheduler() { - return periodScheduler; - } - - private void readLock() { - this.rwLock.readLock().lock(); - } - - private void readUnlock() { - this.rwLock.readLock().unlock(); - } - - private void writeLock() { - this.rwLock.writeLock().lock(); - } - - private void writeUnlock() { - this.rwLock.writeLock().unlock(); - } - - public long write(DataOutputStream dos, long checksum) throws IOException { - MTMVCheckpointData data = new MTMVCheckpointData(); - data.jobs = new ArrayList<>(nameToJobMap.values()); - data.tasks = Lists.newArrayList(taskManager.getHistoryTasks()); - String s = GsonUtils.GSON.toJson(data); - Text.writeString(dos, s); - return checksum; - } - - public static MTMVJobManager read(DataInputStream dis, long checksum) throws IOException { - MTMVJobManager mtmvJobManager = new MTMVJobManager(); - String s = Text.readString(dis); - MTMVCheckpointData data = GsonUtils.GSON.fromJson(s, MTMVCheckpointData.class); - if (data != null) { - if (data.jobs != null) { - for (MTMVJob job : data.jobs) { - mtmvJobManager.replayCreateJob(job); - } - } - - if (data.tasks != null) { - for (MTMVTask runStatus : data.tasks) { - mtmvJobManager.replayCreateJobTask(runStatus); - } - } - } - return mtmvJobManager; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskContext.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskContext.java deleted file mode 100644 index af6f322761..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskContext.java +++ /dev/null @@ -1,72 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.mtmv; - -import org.apache.doris.mtmv.metadata.MTMVJob; -import org.apache.doris.mtmv.metadata.MTMVTask; -import org.apache.doris.qe.ConnectContext; - -import java.util.Map; - -public class MTMVTaskContext { - ConnectContext ctx; - String query; - Map properties; - MTMVTask task; - MTMVJob job; - - public ConnectContext getCtx() { - return ctx; - } - - public void setTask(MTMVTask task) { - this.task = task; - } - - public MTMVTask getTask() { - return this.task; - } - - public void setJob(MTMVJob job) { - this.job = job; - } - - public MTMVJob getJob() { - return this.job; - } - - public void setCtx(ConnectContext ctx) { - this.ctx = ctx; - } - - public String getQuery() { - return query; - } - - public void setQuery(String query) { - this.query = query; - } - - public Map getProperties() { - return properties; - } - - public void setProperties(Map properties) { - this.properties = properties; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskExecuteParams.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskExecuteParams.java deleted file mode 100644 index c0d15cddab..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskExecuteParams.java +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.mtmv; - -import org.apache.doris.mtmv.MTMVUtils.TaskPriority; - -public class MTMVTaskExecuteParams { - private int priority = TaskPriority.LOW.value(); - - public MTMVTaskExecuteParams() { - - } - - public MTMVTaskExecuteParams(int priority) { - this.priority = priority; - } - - public int getPriority() { - return priority; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskExecutor.java deleted file mode 100644 index fb2aaf5419..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskExecutor.java +++ /dev/null @@ -1,181 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.mtmv; - -import org.apache.doris.analysis.UserIdentity; -import org.apache.doris.catalog.Env; -import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.Config; -import org.apache.doris.mtmv.metadata.MTMVJob; -import org.apache.doris.mtmv.metadata.MTMVTask; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.thrift.TUniqueId; - -import com.google.common.collect.Maps; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; - -import java.util.Map; -import java.util.Objects; -import java.util.UUID; -import java.util.concurrent.Future; - -public class MTMVTaskExecutor implements Comparable { - private static final Logger LOG = LogManager.getLogger(MTMVTaskExecutor.class); - - private long jobId; - - private Map properties; - - private Future future; - - private MTMVJob job; - - private ConnectContext ctx; - - private MTMVTaskProcessor processor; - - private MTMVTask task; - - public long getJobId() { - return jobId; - } - - public void setJobId(long jobId) { - this.jobId = jobId; - } - - public MTMVJob getJob() { - return job; - } - - public void setJob(MTMVJob job) { - this.job = job; - } - - public Map getProperties() { - return properties; - } - - public void setProperties(Map properties) { - this.properties = properties; - } - - public Future getFuture() { - return future; - } - - public void setFuture(Future future) { - this.future = future; - } - - public MTMVTaskProcessor getProcessor() { - return processor; - } - - public void setProcessor(MTMVTaskProcessor processor) { - this.processor = processor; - } - - public boolean executeTask() throws Exception { - MTMVTaskContext taskContext = new MTMVTaskContext(); - taskContext.setQuery(task.getQuery()); - ctx = new ConnectContext(); - ctx.setEnv(Env.getCurrentEnv()); - ctx.setCluster(ClusterNamespace.getClusterNameFromFullName(job.getDBName())); - ctx.setDatabase(job.getDBName()); - ctx.setQualifiedUser(task.getUser()); - ctx.setCurrentUserIdentity(UserIdentity.createAnalyzedUserIdentWithIp(job.getUser(), "%")); - ctx.getState().reset(); - UUID taskId = UUID.fromString(task.getTaskId()); - TUniqueId queryId = new TUniqueId(taskId.getMostSignificantBits(), taskId.getLeastSignificantBits()); - ctx.setQueryId(queryId); - - taskContext.setCtx(ctx); - taskContext.setTask(task); - taskContext.setJob(job); - - Map properties = Maps.newHashMap(); - taskContext.setProperties(properties); - return processor.process(taskContext); - } - - public ConnectContext getCtx() { - return ctx; - } - - public MTMVTask getTask() { - return task; - } - - public void setTask(MTMVTask task) { - this.task = task; - } - - public MTMVTask initTask(String taskId, Long createTime) { - MTMVTask task = new MTMVTask(); - task.setTaskId(taskId); - task.setJobName(job.getName()); - if (createTime == null) { - task.setCreateTime(MTMVUtils.getNowTimeStamp()); - } else { - task.setCreateTime(createTime); - } - task.setMVName(job.getMVName()); - task.setUser(job.getUser()); - task.setDBName(job.getDBName()); - task.setQuery(job.getQuery()); - task.setExpireTime(MTMVUtils.getNowTimeStamp() + Config.scheduler_mtmv_task_expired); - task.setRetryTimes(job.getRetryPolicy().getTimes()); - this.task = task; - return task; - } - - public void stop() { - if (ctx != null) { - ctx.kill(false); - } - } - - @Override - public int compareTo(@NotNull MTMVTaskExecutor task) { - if (this.getTask().getPriority() != task.getTask().getPriority()) { - return task.getTask().getPriority() - this.getTask().getPriority(); - } else { - return this.getTask().getCreateTime() > task.getTask().getCreateTime() ? 1 : -1; - } - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - MTMVTaskExecutor task = (MTMVTaskExecutor) o; - return this.task.getQuery().equals(task.getTask().getQuery()); - } - - @Override - public int hashCode() { - return Objects.hash(task); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskExecutorPool.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskExecutorPool.java deleted file mode 100644 index 0c63c287e1..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskExecutorPool.java +++ /dev/null @@ -1,82 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.mtmv; - -import org.apache.doris.mtmv.MTMVUtils.TaskState; -import org.apache.doris.mtmv.metadata.MTMVTask; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; - -public class MTMVTaskExecutorPool { - private static final Logger LOG = LogManager.getLogger(MTMVTaskExecutorPool.class); - private static final long RETRY_INTERVAL = TimeUnit.SECONDS.toMillis(30); - private final ExecutorService taskPool = Executors.newCachedThreadPool(); - - public void executeTask(MTMVTaskExecutor taskExecutor) { - if (taskExecutor == null) { - return; - } - MTMVTask task = taskExecutor.getTask(); - if (task == null) { - return; - } - if (task.getState() == TaskState.SUCCESS || task.getState() == TaskState.FAILURE) { - LOG.warn("Task {} is in final status {} ", task.getTaskId(), task.getState()); - return; - } - - Future future = taskPool.submit(() -> { - task.setState(TaskState.RUNNING); - int retryTimes = task.getRetryTimes(); - boolean isSuccess = false; - do { - try { - isSuccess = taskExecutor.executeTask(); - if (isSuccess) { - task.setState(TaskState.SUCCESS); - break; - } - } catch (Throwable t) { - LOG.warn("Failed to execute the task, taskId=" + task.getTaskId() + ".", t); - } - retryTimes--; - - if (retryTimes > 0) { - try { - Thread.sleep(RETRY_INTERVAL); - } catch (InterruptedException e) { - LOG.warn("Failed to sleep.", e); - break; - } - } - } while (!isSuccess && retryTimes > 0); - if (!isSuccess) { - task.setState(TaskState.FAILURE); - task.setErrorCode(-1); - } - task.setFinishTime(MTMVUtils.getNowTimeStamp()); - }); - taskExecutor.setFuture(future); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskManager.java deleted file mode 100644 index 138ede9e07..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskManager.java +++ /dev/null @@ -1,350 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.mtmv; - -import org.apache.doris.catalog.Env; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Config; -import org.apache.doris.common.CustomThreadFactory; -import org.apache.doris.mtmv.MTMVUtils.TaskState; -import org.apache.doris.mtmv.metadata.MTMVJob; -import org.apache.doris.mtmv.metadata.MTMVTask; - -import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Queues; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.Deque; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.PriorityBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.Collectors; - -public class MTMVTaskManager { - - private static final Logger LOG = LogManager.getLogger(MTMVTaskManager.class); - - // jobId -> pending tasks, one job can dispatch many tasks - private final Map> pendingTaskMap = Maps.newConcurrentMap(); - - // jobId -> running tasks, only one task will be running for one job. - private final Map runningTaskMap = Maps.newConcurrentMap(); - - private final MTMVTaskExecutorPool taskExecutorPool = new MTMVTaskExecutorPool(); - - private final ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true); - - // keep track of all the completed tasks - private final Deque historyTasks = Queues.newLinkedBlockingDeque(); - - private ScheduledExecutorService taskScheduler = Executors.newScheduledThreadPool(1, - new CustomThreadFactory("mtmv-task-scheduler")); - - private final AtomicInteger failedTaskCount = new AtomicInteger(0); - - public void startTaskScheduler() { - if (taskScheduler.isShutdown()) { - taskScheduler = Executors.newScheduledThreadPool(1, new CustomThreadFactory("mtmv-task-scheduler")); - } - taskScheduler.scheduleAtFixedRate(() -> { - checkRunningTask(); - scheduledPendingTask(); - - }, 0, 1, TimeUnit.SECONDS); - } - - public void stopTaskScheduler() { - taskScheduler.shutdown(); - } - - private void checkRunningTask() { - writeLock(); - try { - Iterator runningIterator = runningTaskMap.keySet().iterator(); - while (runningIterator.hasNext()) { - Long jobId = runningIterator.next(); - MTMVTaskExecutor taskExecutor = runningTaskMap.get(jobId); - Future future = taskExecutor.getFuture(); - if (future.isDone()) { - runningIterator.remove(); - addHistory(taskExecutor.getTask()); - MTMVUtils.TaskState finalState = taskExecutor.getTask().getState(); - if (finalState == TaskState.FAILURE) { - failedTaskCount.incrementAndGet(); - } - //task save final state only - Env.getCurrentEnv().getEditLog().logCreateMTMVTask(taskExecutor.getTask()); - taskExecutor.getJob().taskFinished(); - } - } - } finally { - writeUnlock(); - } - } - - private void scheduledPendingTask() { - writeLock(); - try { - int currentRunning = runningTaskMap.size(); - - Iterator pendingIterator = pendingTaskMap.keySet().iterator(); - while (pendingIterator.hasNext()) { - Long jobId = pendingIterator.next(); - MTMVTaskExecutor runningTaskExecutor = runningTaskMap.get(jobId); - if (runningTaskExecutor == null) { - Queue taskQueue = pendingTaskMap.get(jobId); - if (taskQueue.size() == 0) { - pendingIterator.remove(); - } else { - if (currentRunning >= Config.max_running_mtmv_scheduler_task_num) { - break; - } - MTMVTaskExecutor pendingTaskExecutor = taskQueue.poll(); - taskExecutorPool.executeTask(pendingTaskExecutor); - runningTaskMap.put(jobId, pendingTaskExecutor); - currentRunning++; - } - } - } - } finally { - writeUnlock(); - } - } - - public MTMVUtils.TaskSubmitStatus submitJobTask(MTMVJob job) { - return submitJobTask(job, new MTMVTaskExecuteParams()); - } - - private MTMVUtils.TaskSubmitStatus submitJobTask(MTMVJob job, MTMVTaskExecuteParams param) { - return submitTask(MTMVUtils.buildTask(job), param); - } - - private MTMVUtils.TaskSubmitStatus submitTask(MTMVTaskExecutor taskExecutor, MTMVTaskExecuteParams params) { - // duplicate submit - if (taskExecutor.getTask() != null) { - return MTMVUtils.TaskSubmitStatus.FAILED; - } - - int validPendingCount = 0; - for (Long jobId : pendingTaskMap.keySet()) { - if (!pendingTaskMap.get(jobId).isEmpty()) { - validPendingCount++; - } - } - - if (validPendingCount >= Config.max_pending_mtmv_scheduler_task_num) { - LOG.warn("pending task exceeds pending_scheduler_task_size:{}, reject the submit.", - Config.max_pending_mtmv_scheduler_task_num); - return MTMVUtils.TaskSubmitStatus.REJECTED; - } - - String taskId = UUID.randomUUID().toString(); - MTMVTask task = taskExecutor.initTask(taskId, MTMVUtils.getNowTimeStamp()); - task.setPriority(params.getPriority()); - LOG.info("Submit a mtmv task with id: {} of the job {}.", taskId, taskExecutor.getJob().getName()); - arrangeToPendingTask(taskExecutor); - return MTMVUtils.TaskSubmitStatus.SUBMITTED; - } - - private void arrangeToPendingTask(MTMVTaskExecutor task) { - writeLock(); - try { - long jobId = task.getJobId(); - PriorityBlockingQueue tasks = - pendingTaskMap.computeIfAbsent(jobId, u -> Queues.newPriorityBlockingQueue()); - tasks.offer(task); - } finally { - writeUnlock(); - } - } - - public void dealJobRemoved(MTMVJob job) { - removePendingTask(job.getId()); - removeRunningTask(job.getId()); - if (!Config.keep_scheduler_mtmv_task_when_job_deleted) { - clearHistoryTasksByJobName(job.getName(), false); - } - } - - private void removePendingTask(Long jobId) { - pendingTaskMap.remove(jobId); - } - - private void removeRunningTask(Long jobId) { - MTMVTaskExecutor task = runningTaskMap.remove(jobId); - if (task != null) { - task.stop(); - } - } - - public int getFailedTaskCount() { - return failedTaskCount.get(); - } - - public Map> getPendingTaskMap() { - return pendingTaskMap; - } - - public Map getRunningTaskMap() { - return runningTaskMap; - } - - private void addHistory(MTMVTask task) { - historyTasks.addFirst(task); - } - - public Deque getHistoryTasks() { - return historyTasks; - } - - public List getHistoryTasksByJobName(String jobName) { - return getHistoryTasks().stream().filter(u -> u.getJobName().equals(jobName)) - .collect(Collectors.toList()); - } - - public List showAllTasks() { - return showTasksWithLock(null); - } - - public List showTasksWithLock(String dbName) { - List taskList = Lists.newArrayList(); - readLock(); - try { - if (Strings.isNullOrEmpty(dbName)) { - for (Queue pTaskQueue : getPendingTaskMap().values()) { - taskList.addAll(pTaskQueue.stream().map(MTMVTaskExecutor::getTask).collect(Collectors.toList())); - } - taskList.addAll( - getRunningTaskMap().values().stream().map(MTMVTaskExecutor::getTask) - .collect(Collectors.toList())); - taskList.addAll(getHistoryTasks()); - } else { - for (Queue pTaskQueue : getPendingTaskMap().values()) { - taskList.addAll( - pTaskQueue.stream().map(MTMVTaskExecutor::getTask).filter(u -> u.getDBName().equals(dbName)) - .collect(Collectors.toList())); - } - taskList.addAll(getRunningTaskMap().values().stream().map(MTMVTaskExecutor::getTask) - .filter(u -> u.getDBName().equals(dbName)).collect(Collectors.toList())); - taskList.addAll( - getHistoryTasks().stream().filter(u -> u.getDBName().equals(dbName)) - .collect(Collectors.toList())); - - } - } finally { - readUnlock(); - } - - return taskList.stream().sorted().collect(Collectors.toList()); - } - - public List showTasks(String dbName, String mvName) { - return showTasksWithLock(dbName).stream().filter(u -> u.getMVName().equals(mvName)) - .collect(Collectors.toList()); - } - - public MTMVTask getTask(String taskId) throws AnalysisException { - List tasks = - showAllTasks().stream().filter(u -> u.getTaskId().equals(taskId)).collect(Collectors.toList()); - if (tasks.size() == 0) { - throw new AnalysisException("Can't find the task id in the task list."); - } else if (tasks.size() > 1) { - throw new AnalysisException("Find more than one task id in the task list."); - } else { - return tasks.get(0); - } - } - - public void replayCreateJobTask(MTMVTask task) { - addHistory(task); - } - - private void clearHistoryTasksByJobName(String jobName, boolean isReplay) { - List clearTasks = Lists.newArrayList(); - - Deque taskHistory = getHistoryTasks(); - for (MTMVTask task : taskHistory) { - if (task.getJobName().equals(jobName)) { - clearTasks.add(task.getTaskId()); - } - } - - dropHistoryTasks(clearTasks, isReplay); - } - - public void removeExpiredTasks() { - long currentTime = MTMVUtils.getNowTimeStamp(); - List historyToDelete = Lists.newArrayList(); - Deque taskHistory = getHistoryTasks(); - for (MTMVTask task : taskHistory) { - long expireTime = task.getExpireTime(); - if (currentTime > expireTime) { - historyToDelete.add(task.getTaskId()); - } - } - dropHistoryTasks(historyToDelete, false); - } - - public void dropHistoryTasks(List taskIds, boolean isReplay) { - if (taskIds.isEmpty()) { - return; - } - writeLock(); - try { - Set taskSet = new HashSet<>(taskIds); - // Try to remove history tasks. - getHistoryTasks().removeIf(mtmvTask -> taskSet.contains(mtmvTask.getTaskId())); - if (!isReplay) { - Env.getCurrentEnv().getEditLog().logDropMTMVTasks(taskIds); - } - } finally { - writeUnlock(); - } - LOG.info("drop task history:{}", taskIds); - } - - private void readLock() { - this.rwLock.readLock().lock(); - } - - private void readUnlock() { - this.rwLock.readLock().unlock(); - } - - private void writeLock() { - this.rwLock.writeLock().lock(); - } - - private void writeUnlock() { - this.rwLock.writeLock().unlock(); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskProcessor.java deleted file mode 100644 index 0e12c294e1..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVTaskProcessor.java +++ /dev/null @@ -1,158 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.mtmv; - -import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.MaterializedView; -import org.apache.doris.common.ErrorCode; -import org.apache.doris.common.FeConstants; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.QueryState; -import org.apache.doris.qe.QueryState.MysqlStateType; -import org.apache.doris.qe.StmtExecutor; - -import com.google.common.collect.Lists; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.List; -import java.util.concurrent.atomic.AtomicLong; - - -public class MTMVTaskProcessor { - private static final Logger LOG = LogManager.getLogger(MTMVTaskProcessor.class); - private static final AtomicLong STMT_ID_GENERATOR = new AtomicLong(0); - - boolean process(MTMVTaskContext context) throws Exception { - String taskId = context.getTask().getTaskId(); - long jobId = context.getJob().getId(); - LOG.info("Start to run a MTMV task, taskId={}, jobId={}.", taskId, jobId); - - String mvName = context.getTask().getMVName(); - String temporaryMVName = getTemporaryMVName(mvName); - Database db = context.getCtx().getEnv().getInternalCatalog() - .getDbOrMetaException(context.getTask().getDBName()); - MaterializedView mv = (MaterializedView) db.getTableOrAnalysisException(mvName); - - if (!mv.tryLockMVTask()) { - LOG.warn("Failed to run the MTMV task, taskId={}, jobId={}, msg={}.", taskId, jobId, - "Failed to get the lock"); - context.getTask().setMessage("Failed to get the lock."); - return false; - } - try { - // Check whether the temporary materialized view exists, we should drop the obsolete materialized view first - // because it was created by previous tasks which failed to complete their work. - dropMaterializedView(context, temporaryMVName); - - // Step 1: create the temporary materialized view. - String createStatement = generateCreateStatement(mv.clone(temporaryMVName)); - if (!executeSQL(context, createStatement)) { - throw new RuntimeException( - "Failed to create the temporary materialized view, sql=" + createStatement + ", cause=" - + context.getCtx().getState().getErrorMessage() + "."); - } - - // Step 2: insert data to the temporary materialized view. - String insertSelectStatement = generateInsertSelectStmt(context, temporaryMVName); - if (!executeSQL(context, insertSelectStatement)) { - throw new RuntimeException( - "Failed to insert data to the temporary materialized view, sql=" + insertSelectStatement - + ", cause=" + context.getCtx().getState().getErrorMessage() + "."); - } - String insertInfoMessage = context.getCtx().getState().getInfoMessage(); - - // Step 3: swap the temporary materialized view with the original materialized view. - String swapStatement = generateSwapStatement(mvName, temporaryMVName); - if (!executeSQL(context, swapStatement)) { - throw new RuntimeException( - "Failed to swap the temporary materialized view with the original materialized view, sql=" - + swapStatement + ", cause=" + context.getCtx().getState().getErrorMessage() + "."); - } - - context.getTask().setMessage(insertInfoMessage); - LOG.info("Run MTMV task successfully, taskId={}, jobId={}.", taskId, jobId); - return true; - } catch (Throwable e) { - context.getTask().setMessage(e.getMessage()); - throw e; - } finally { - mv.unLockMVTask(); - dropMaterializedView(context, temporaryMVName); - } - } - - private String getTemporaryMVName(String mvName) { - return FeConstants.TEMP_MATERIZLIZE_DVIEW_PREFIX + mvName; - } - - private void dropMaterializedView(MTMVTaskContext context, String mvName) { - String dropStatement = generateDropStatement(mvName); - if (!executeSQL(context, dropStatement)) { - throw new RuntimeException( - "Failed to drop the temporary materialized view, sql=" + dropStatement + "."); - } - } - - private String generateDropStatement(String mvName) { - return "DROP MATERIALIZED VIEW IF EXISTS " + mvName; - } - - private boolean executeSQL(MTMVTaskContext context, String sql) { - ConnectContext ctx = context.getCtx(); - ctx.setThreadLocalInfo(); - ctx.getState().reset(); - try { - ctx.getSessionVariable().disableNereidsPlannerOnce(); - StmtExecutor executor = new StmtExecutor(ctx, sql); - ctx.setExecutor(executor); - executor.execute(); - } catch (Throwable e) { - QueryState queryState = new QueryState(); - queryState.setError(ErrorCode.ERR_INTERNAL_ERROR, e.getMessage()); - ctx.setState(queryState); - } finally { - ConnectContext.remove(); - } - - if (ctx.getState().getStateType() == MysqlStateType.OK) { - LOG.info("Execute SQL successfully, taskId={}, sql={}.", context.getTask().getTaskId(), sql); - } else { - LOG.warn("Failed to execute SQL, taskId={}, sql={}, errorCode={}, message={}.", - context.getTask().getTaskId(), - sql, ctx.getState().getErrorCode(), ctx.getState().getErrorMessage()); - } - return ctx.getState().getStateType() == MysqlStateType.OK; - } - - private String generateCreateStatement(MaterializedView mv) { - List createStatement = Lists.newArrayList(); - Env.getDdlStmt(mv, createStatement, null, null, false, true /* hide password */, -1L); - return createStatement.stream().findFirst().orElse(""); - } - - private String generateInsertSelectStmt(MTMVTaskContext context, String temporaryMVName) { - return "INSERT INTO " + temporaryMVName + " " + context.getQuery(); - } - - // ALTER TABLE t1 REPLACE WITH TABLE t1_mirror PROPERTIES('swap' = 'false'); - private String generateSwapStatement(String mvName, String temporaryMVName) { - return "ALTER TABLE " + mvName + " REPLACE WITH TABLE " + temporaryMVName + " PROPERTIES('swap' = 'false')"; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtils.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtils.java deleted file mode 100644 index ffaf5af888..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtils.java +++ /dev/null @@ -1,137 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.mtmv; - -import org.apache.doris.mtmv.metadata.MTMVJob; - -import java.time.Duration; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.concurrent.TimeUnit; - -public class MTMVUtils { - - public enum TriggerMode { - MANUAL, ONCE, PERIODICAL, ON_COMMIT - } - - /** - * All the job init status: UNKNOWN - * create job: UNKNOWN -> ACTIVE - * after job scheduler: - * - ONCE Job: ACTIVE -> COMPLETE (no matter task success or fail) - * - PERIODICAL Job: ACTIVE -> PAUSE (if the task failed too many times) - * - PERIODICAL Job: ACTIVE -> COMPLETE (when the job is expired.) - * recover job: PAUSE -> ACTIVE (specific command) - */ - public enum JobState { - UNKNOWN, ACTIVE, PAUSE, COMPLETE - } - - public enum TaskRetryPolicy { - NEVER(0), // no any retry - TIMES(3), // fix 3 times retry - ALWAYS(100); // treat 100 as big number since most case will cause a lot of resource - private final int times; - - TaskRetryPolicy(int times) { - this.times = times; - } - - public int getTimes() { - return times; - } - } - - public enum TaskState { - PENDING, RUNNING, FAILURE, SUCCESS, - } - - public enum TaskSubmitStatus { - SUBMITTED, REJECTED, FAILED - } - - public enum TaskPriority { - LOW(0), NORMAL(50), HIGH(100); - - private final int value; - - TaskPriority(int value) { - this.value = value; - } - - public int value() { - return value; - } - } - - public static long getDelaySeconds(MTMVJob job) { - return getDelaySeconds(job, LocalDateTime.now()); - } - - public static long getDelaySeconds(MTMVJob job, LocalDateTime now) { - long lastModifyTime = job.getLastModifyTime(); - long nextTime = 0; - // if set lastModifyTime, use lastModifyTime otherwise use the start time. - if (lastModifyTime > 0) { - // check null of schedule outside. - nextTime = lastModifyTime + job.getSchedule().getSecondPeriod(); - } else { - nextTime = job.getSchedule().getStartTime(); - } - LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochSecond(nextTime), ZoneId.systemDefault()); - Duration duration = Duration.between(now, time); - long delaySeconds = duration.getSeconds(); - // start immediately if startTime < now - return delaySeconds < 0 ? 0 : delaySeconds; - } - - public static MTMVTaskExecutor buildTask(MTMVJob job) { - MTMVTaskExecutor taskExecutor = new MTMVTaskExecutor(); - taskExecutor.setJobId(job.getId()); - taskExecutor.setProperties(job.getProperties()); - taskExecutor.setJob(job); - taskExecutor.setProcessor(new MTMVTaskProcessor()); - - return taskExecutor; - } - - public static TimeUnit getTimeUint(String strTimeUnit) { - switch (strTimeUnit.toUpperCase()) { - case "SECOND": - return TimeUnit.SECONDS; - case "MINUTE": - return TimeUnit.MINUTES; - case "HOUR": - return TimeUnit.HOURS; - default: - return TimeUnit.DAYS; - } - } - - // In MTMV package, all the timestamp unit is second. - public static long getNowTimeStamp() { - return System.currentTimeMillis() / 1000; - } - - public static String getTimeString(long timestamp) { - LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochSecond(timestamp), ZoneId.systemDefault()); - return time.toString(); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/ChangeMTMVJob.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/ChangeMTMVJob.java deleted file mode 100644 index 42388b7070..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/ChangeMTMVJob.java +++ /dev/null @@ -1,99 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.mtmv.metadata; - -import org.apache.doris.common.io.Text; -import org.apache.doris.common.io.Writable; -import org.apache.doris.mtmv.MTMVUtils; -import org.apache.doris.mtmv.MTMVUtils.JobState; -import org.apache.doris.persist.gson.GsonUtils; - -import com.google.gson.annotations.SerializedName; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -public class ChangeMTMVJob implements Writable { - @SerializedName("jobId") - private long jobId; - - @SerializedName("lastModifyTime") - private long lastModifyTime; - - @SerializedName("fromStatus") - JobState fromStatus; - - @SerializedName("toStatus") - JobState toStatus; - - @SerializedName("errorCode") - private int errorCode; - - @SerializedName("errorMessage") - private String errorMessage; - - public ChangeMTMVJob(long jobId, JobState toStatus) { - this.jobId = jobId; - this.toStatus = toStatus; - this.lastModifyTime = MTMVUtils.getNowTimeStamp(); - } - - public long getJobId() { - return jobId; - } - - public void setJobId(long jobId) { - this.jobId = jobId; - } - - public long getLastModifyTime() { - return lastModifyTime; - } - - public void setLastModifyTime(long lastModifyTime) { - this.lastModifyTime = lastModifyTime; - } - - public JobState getFromStatus() { - return fromStatus; - } - - public void setFromStatus(JobState fromStatus) { - this.fromStatus = fromStatus; - } - - public JobState getToStatus() { - return toStatus; - } - - public void setToStatus(JobState toStatus) { - this.toStatus = toStatus; - } - - public static ChangeMTMVJob read(DataInput in) throws IOException { - String json = Text.readString(in); - return GsonUtils.GSON.fromJson(json, ChangeMTMVJob.class); - } - - @Override - public void write(DataOutput out) throws IOException { - String json = GsonUtils.GSON.toJson(this); - Text.writeString(out, json); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/DropMTMVJob.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/DropMTMVJob.java deleted file mode 100644 index c6b21b53b0..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/DropMTMVJob.java +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.mtmv.metadata; - -import org.apache.doris.common.io.Text; -import org.apache.doris.common.io.Writable; -import org.apache.doris.persist.gson.GsonUtils; - -import com.google.gson.annotations.SerializedName; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.List; - -public class DropMTMVJob implements Writable { - @SerializedName("jobIds") - List jobIds; - - public DropMTMVJob(List jobIds) { - this.jobIds = jobIds; - } - - public List getJobIds() { - return jobIds; - } - - public static DropMTMVJob read(DataInput in) throws IOException { - return GsonUtils.GSON.fromJson(Text.readString(in), DropMTMVJob.class); - } - - @Override - public void write(DataOutput out) throws IOException { - String json = GsonUtils.GSON.toJson(this); - Text.writeString(out, json); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/DropMTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/DropMTMVTask.java deleted file mode 100644 index 3c5848f330..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/DropMTMVTask.java +++ /dev/null @@ -1,53 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.mtmv.metadata; - -import org.apache.doris.common.io.Text; -import org.apache.doris.common.io.Writable; -import org.apache.doris.persist.gson.GsonUtils; - -import com.google.gson.annotations.SerializedName; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.List; - -public class DropMTMVTask implements Writable { - - @SerializedName("taskIds") - List taskIds; - - public DropMTMVTask(List taskIdList) { - this.taskIds = taskIdList; - } - - public List getTaskIds() { - return taskIds; - } - - public static DropMTMVTask read(DataInput in) throws IOException { - return GsonUtils.GSON.fromJson(Text.readString(in), DropMTMVTask.class); - } - - @Override - public void write(DataOutput out) throws IOException { - String json = GsonUtils.GSON.toJson(this); - Text.writeString(out, json); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/MTMVCheckpointData.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/MTMVCheckpointData.java deleted file mode 100644 index 933ce31cf2..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/MTMVCheckpointData.java +++ /dev/null @@ -1,30 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.mtmv.metadata; - -import com.google.gson.annotations.SerializedName; - -import java.util.List; - -public class MTMVCheckpointData { - @SerializedName("jobs") - public List jobs; - - @SerializedName("tasks") - public List tasks; -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/MTMVJob.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/MTMVJob.java deleted file mode 100644 index f7e2cadeb6..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/MTMVJob.java +++ /dev/null @@ -1,364 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.mtmv.metadata; - -import org.apache.doris.catalog.Env; -import org.apache.doris.common.io.Text; -import org.apache.doris.common.io.Writable; -import org.apache.doris.mtmv.MTMVUtils; -import org.apache.doris.mtmv.MTMVUtils.JobState; -import org.apache.doris.mtmv.MTMVUtils.TaskRetryPolicy; -import org.apache.doris.mtmv.MTMVUtils.TriggerMode; -import org.apache.doris.persist.gson.GsonUtils; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.gson.annotations.SerializedName; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -public class MTMVJob implements Writable, Comparable { - private static final Logger LOG = LogManager.getLogger(MTMVJob.class); - - @SerializedName("id") - private long id; - - @SerializedName("name") - private String name; - - // set default to MANUAL is for compatibility - @SerializedName("triggerMode") - private MTMVUtils.TriggerMode triggerMode = MTMVUtils.TriggerMode.MANUAL; - - @SerializedName("state") - private MTMVUtils.JobState state = JobState.ACTIVE; - - @SerializedName("schedule") - private JobSchedule schedule; - - @SerializedName("createTime") - private long createTime; - - @SerializedName("dbName") - private String dbName; - - @SerializedName("mvName") - private String mvName; - - @SerializedName("query") - private String query; - - @SerializedName("properties") - private Map properties; - - @SerializedName("expireTime") - private long expireTime = -1; - - // set default to ROOT is for compatibility - @SerializedName("user") - private String user = "root"; - - @SerializedName("retryPolicy") - private TaskRetryPolicy retryPolicy = TaskRetryPolicy.NEVER; - - @SerializedName("lastModifyTime") - private long lastModifyTime; - - private ScheduledFuture future; - - public MTMVJob(String name) { - this.name = name; - this.createTime = MTMVUtils.getNowTimeStamp(); - } - - public static MTMVJob read(DataInput in) throws IOException { - String json = Text.readString(in); - return GsonUtils.GSON.fromJson(json, MTMVJob.class); - } - - @Override - public void write(DataOutput out) throws IOException { - String json = GsonUtils.GSON.toJson(this); - Text.writeString(out, json); - } - - public long getId() { - return id; - } - - public void setId(long id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public TriggerMode getTriggerMode() { - return triggerMode; - } - - public void setTriggerMode(TriggerMode triggerMode) { - this.triggerMode = triggerMode; - } - - public JobState getState() { - return state; - } - - public void setState(JobState state) { - this.state = state; - } - - public JobSchedule getSchedule() { - return schedule; - } - - public void setSchedule(JobSchedule schedule) { - this.schedule = schedule; - } - - public long getCreateTime() { - return createTime; - } - - public void setCreateTime(long createTime) { - this.createTime = createTime; - } - - public String getDBName() { - return dbName; - } - - public void setDBName(String dbName) { - this.dbName = dbName; - } - - public String getMVName() { - return mvName; - } - - public void setMVName(String mvName) { - this.mvName = mvName; - } - - public String getQuery() { - return query == null ? "" : query; - } - - public void setQuery(String query) { - this.query = query; - } - - public Map getProperties() { - return properties; - } - - public void setProperties(Map properties) { - this.properties = properties; - } - - public long getExpireTime() { - return expireTime; - } - - public void setExpireTime(long expireTime) { - this.expireTime = expireTime; - } - - public String getUser() { - return user; - } - - public void setUser(String user) { - this.user = user; - } - - public TaskRetryPolicy getRetryPolicy() { - return retryPolicy; - } - - public void setRetryPolicy(TaskRetryPolicy retryPolicy) { - this.retryPolicy = retryPolicy; - } - - public long getLastModifyTime() { - return lastModifyTime; - } - - public void setLastModifyTime(long lastModifyTime) { - this.lastModifyTime = lastModifyTime; - } - - public static class JobSchedule { - @SerializedName("startTime") - private long startTime; // second - - @SerializedName("period") - private long period; - - @SerializedName("timeUnit") - private TimeUnit timeUnit; - - public JobSchedule(long startTime, long period, TimeUnit timeUnit) { - this.startTime = startTime; - this.period = period; - this.timeUnit = timeUnit; - } - - public long getStartTime() { - return startTime; - } - - public void setStartTime(long startTime) { - this.startTime = startTime; - } - - public long getPeriod() { - return period; - } - - public void setPeriod(long period) { - this.period = period; - } - - public TimeUnit getTimeUnit() { - return timeUnit; - } - - public void setTimeUnit(TimeUnit timeUnit) { - this.timeUnit = timeUnit; - } - - public long getSecondPeriod() { - return getTimeUnit().toSeconds(getPeriod()); - } - - public String toString() { - return "START " + LocalDateTime.ofInstant(Instant.ofEpochSecond(startTime), ZoneId.systemDefault()) - + " EVERY(" + period + " " + timeUnit + ")"; - } - } - - public static final ImmutableList SHOW_TITLE_NAMES = - new ImmutableList.Builder() - .add("Id") - .add("Name") - .add("TriggerMode") - .add("Schedule") - .add("DBName") - .add("MVName") - .add("Query") - .add("User") - .add("RetryPolicy") - .add("State") - .add("CreateTime") - .add("ExpireTime") - .add("LastModifyTime") - .build(); - - public List toStringRow() { - List list = Lists.newArrayList(); - list.add(Long.toString(getId())); - list.add(getName()); - list.add(getTriggerMode().toString()); - list.add(getSchedule() == null ? "NULL" : getSchedule().toString()); - list.add(getDBName()); - list.add(getMVName()); - list.add(getQuery().length() > 10240 ? getQuery().substring(0, 10240) : getQuery()); - list.add(getUser()); - list.add(getRetryPolicy().toString()); - list.add(getState().toString()); - list.add(MTMVUtils.getTimeString(getCreateTime())); - list.add(MTMVUtils.getTimeString(getExpireTime())); - list.add(MTMVUtils.getTimeString(getLastModifyTime())); - return list; - } - - public synchronized void start() { - - if (state == JobState.COMPLETE || state == JobState.PAUSE) { - return; - } - if (getTriggerMode() == TriggerMode.PERIODICAL) { - JobSchedule schedule = getSchedule(); - ScheduledExecutorService periodScheduler = Env.getCurrentEnv().getMTMVJobManager().getPeriodScheduler(); - future = periodScheduler.scheduleAtFixedRate( - () -> Env.getCurrentEnv().getMTMVJobManager().getTaskManager().submitJobTask(this), - MTMVUtils.getDelaySeconds(this), schedule.getSecondPeriod(), TimeUnit.SECONDS); - - } else if (getTriggerMode() == TriggerMode.ONCE) { - Env.getCurrentEnv().getMTMVJobManager().getTaskManager().submitJobTask(this); - } - } - - public synchronized void stop() { - // MUST not set true for "mayInterruptIfRunning". - // Because this thread may doing bdbje write operation, it is interrupted, - // FE may exit due to bdbje write failure. - if (future != null) { - boolean isCancel = future.cancel(false); - if (!isCancel) { - LOG.warn("fail to cancel scheduler for job [{}]", name); - } - } - Env.getCurrentEnv().getMTMVJobManager().getTaskManager().dealJobRemoved(this); - } - - public void taskFinished() { - if (triggerMode == TriggerMode.ONCE) { - // update the run once job status - ChangeMTMVJob changeJob = new ChangeMTMVJob(id, JobState.COMPLETE); - updateJob(changeJob, false); - } else if (triggerMode == TriggerMode.PERIODICAL) { - // just update the last modify time. - ChangeMTMVJob changeJob = new ChangeMTMVJob(id, JobState.ACTIVE); - updateJob(changeJob, false); - } - } - - public void updateJob(ChangeMTMVJob changeJob, boolean isReplay) { - setState(changeJob.getToStatus()); - setLastModifyTime(changeJob.getLastModifyTime()); - if (!isReplay) { - Env.getCurrentEnv().getEditLog().logChangeMTMVJob(changeJob); - } - } - - @Override - public int compareTo(@NotNull Object o) { - return (int) (getCreateTime() - ((MTMVJob) o).getCreateTime()); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/MTMVTask.java deleted file mode 100644 index 603e98a560..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/metadata/MTMVTask.java +++ /dev/null @@ -1,244 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.mtmv.metadata; - -import org.apache.doris.common.io.Text; -import org.apache.doris.common.io.Writable; -import org.apache.doris.mtmv.MTMVUtils; -import org.apache.doris.persist.gson.GsonUtils; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.gson.annotations.SerializedName; -import org.jetbrains.annotations.NotNull; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.util.List; - -public class MTMVTask implements Writable, Comparable { - // also named query id in ConnectContext - @SerializedName("taskId") - private String taskId; - - @SerializedName("jobName") - private String jobName; - - @SerializedName("createTime") - private long createTime; - - @SerializedName("finishTime") - private long finishTime; - - @SerializedName("state") - private MTMVUtils.TaskState state = MTMVUtils.TaskState.PENDING; - - @SerializedName("dbName") - private String dbName; - - @SerializedName("mvName") - private String mvName; - - @SerializedName("query") - private String query; - - @SerializedName("user") - private String user; - - @SerializedName("message") - private String message; - - @SerializedName("errorCode") - private int errorCode; - - @SerializedName("expireTime") - private long expireTime; - - // the larger the value, the higher the priority, the default value is 0 - @SerializedName("priority") - private int priority = 0; - - @SerializedName("retryTimes") - private int retryTimes = 0; - - public String getTaskId() { - return taskId; - } - - public void setTaskId(String taskId) { - this.taskId = taskId; - } - - public String getJobName() { - return jobName; - } - - public void setJobName(String jobName) { - this.jobName = jobName; - } - - public long getCreateTime() { - return createTime; - } - - public void setCreateTime(long createTime) { - this.createTime = createTime; - } - - public long getFinishTime() { - return finishTime; - } - - public void setFinishTime(long finishTime) { - this.finishTime = finishTime; - } - - public MTMVUtils.TaskState getState() { - return state; - } - - public void setState(MTMVUtils.TaskState state) { - this.state = state; - } - - public String getDBName() { - return dbName; - } - - public void setDBName(String dbName) { - this.dbName = dbName; - } - - public String getMVName() { - return mvName; - } - - public void setMVName(String mvName) { - this.mvName = mvName; - } - - public String getUser() { - return user; - } - - public void setUser(String user) { - this.user = user; - } - - public String getQuery() { - return query == null ? "" : query; - } - - public void setQuery(String query) { - this.query = query; - } - - public int getErrorCode() { - return errorCode; - } - - public void setErrorCode(int errorCode) { - this.errorCode = errorCode; - } - - public String getMessage() { - return message == null ? "" : message; - } - - public void setMessage(String message) { - this.message = message; - } - - public long getExpireTime() { - return expireTime; - } - - public void setExpireTime(long expireTime) { - this.expireTime = expireTime; - } - - public int getPriority() { - return priority; - } - - public void setPriority(int priority) { - this.priority = priority; - } - - public int getRetryTimes() { - return retryTimes; - } - - public void setRetryTimes(int retryTimes) { - this.retryTimes = retryTimes; - } - - public static MTMVTask read(DataInput in) throws IOException { - String json = Text.readString(in); - return GsonUtils.GSON.fromJson(json, MTMVTask.class); - } - - @Override - public void write(DataOutput out) throws IOException { - String json = GsonUtils.GSON.toJson(this); - Text.writeString(out, json); - } - - public static final ImmutableList SHOW_TITLE_NAMES = - new ImmutableList.Builder() - .add("TaskId") - .add("JobName") - .add("DBName") - .add("MVName") - .add("Query") - .add("User") - .add("Priority") - .add("RetryTimes") - .add("State") - .add("Message") - .add("ErrorCode") - .add("CreateTime") - .add("ExpireTime") - .add("FinishTime") - .build(); - - public List toStringRow() { - List list = Lists.newArrayList(); - list.add(getTaskId()); - list.add(getJobName()); - list.add(getDBName()); - list.add(getMVName()); - list.add(getQuery().length() > 10240 ? getQuery().substring(0, 10240) : getQuery()); - list.add(getUser()); - list.add(Integer.toString(getPriority())); - list.add(Integer.toString(getRetryTimes())); - list.add(getState().toString()); - list.add(getMessage().length() > 10240 ? getMessage().substring(0, 10240) : getMessage()); - list.add(Integer.toString(getErrorCode())); - list.add(MTMVUtils.getTimeString(getCreateTime())); - list.add(MTMVUtils.getTimeString(getExpireTime())); - list.add(MTMVUtils.getTimeString(getFinishTime())); - return list; - } - - @Override - public int compareTo(@NotNull Object o) { - return (int) (getCreateTime() - ((MTMVTask) o).getCreateTime()); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterMultiMaterializedView.java b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterMultiMaterializedView.java deleted file mode 100644 index 11c675370e..0000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterMultiMaterializedView.java +++ /dev/null @@ -1,61 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.persist; - -import org.apache.doris.analysis.MVRefreshInfo; -import org.apache.doris.analysis.TableName; -import org.apache.doris.common.io.Text; -import org.apache.doris.common.io.Writable; -import org.apache.doris.persist.gson.GsonUtils; - -import com.google.gson.annotations.SerializedName; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -public class AlterMultiMaterializedView implements Writable { - - @SerializedName(value = "mvName") - private TableName mvName; - - @SerializedName(value = "info") - private MVRefreshInfo info; - - public AlterMultiMaterializedView(TableName mvName, MVRefreshInfo info) { - this.mvName = mvName; - this.info = info; - } - - public MVRefreshInfo getInfo() { - return info; - } - - public TableName getMvName() { - return mvName; - } - - @Override - public void write(DataOutput out) throws IOException { - Text.writeString(out, GsonUtils.GSON.toJson(this)); - } - - public static AlterMultiMaterializedView read(DataInput in) throws IOException { - return GsonUtils.GSON.fromJson(Text.readString(in), AlterMultiMaterializedView.class); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index a81775e4ec..d17fbc99c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -72,11 +72,6 @@ import org.apache.doris.load.routineload.RoutineLoadJob; import org.apache.doris.load.sync.SyncJob; import org.apache.doris.meta.MetaContext; import org.apache.doris.metric.MetricRepo; -import org.apache.doris.mtmv.metadata.ChangeMTMVJob; -import org.apache.doris.mtmv.metadata.DropMTMVJob; -import org.apache.doris.mtmv.metadata.DropMTMVTask; -import org.apache.doris.mtmv.metadata.MTMVJob; -import org.apache.doris.mtmv.metadata.MTMVTask; import org.apache.doris.mysql.privilege.UserPropertyInfo; import org.apache.doris.plugin.PluginInfo; import org.apache.doris.policy.DropPolicyLog; @@ -968,37 +963,13 @@ public class EditLog { env.getLoadManager().replayCleanLabel(log); break; } - case OperationType.OP_CREATE_MTMV_JOB: { - final MTMVJob job = (MTMVJob) journal.getData(); - env.getMTMVJobManager().replayCreateJob(job); - break; - } - case OperationType.OP_CHANGE_MTMV_JOB: { - final ChangeMTMVJob changeJob = (ChangeMTMVJob) journal.getData(); - env.getMTMVJobManager().replayUpdateJob(changeJob); - break; - } - case OperationType.OP_DROP_MTMV_JOB: { - final DropMTMVJob dropJob = (DropMTMVJob) journal.getData(); - env.getMTMVJobManager().replayDropJobs(dropJob.getJobIds()); - break; - } - case OperationType.OP_CREATE_MTMV_TASK: { - final MTMVTask task = (MTMVTask) journal.getData(); - env.getMTMVJobManager().replayCreateJobTask(task); - break; - } - case OperationType.OP_CHANGE_MTMV_TASK: { - break; - } - case OperationType.OP_DROP_MTMV_TASK: { - final DropMTMVTask dropTask = (DropMTMVTask) journal.getData(); - env.getMTMVJobManager().replayDropJobTasks(dropTask.getTaskIds()); - break; - } + case OperationType.OP_CREATE_MTMV_JOB: + case OperationType.OP_CHANGE_MTMV_JOB: + case OperationType.OP_DROP_MTMV_JOB: + case OperationType.OP_CREATE_MTMV_TASK: + case OperationType.OP_CHANGE_MTMV_TASK: + case OperationType.OP_DROP_MTMV_TASK: case OperationType.OP_ALTER_MTMV_STMT: { - final AlterMultiMaterializedView alterView = (AlterMultiMaterializedView) journal.getData(); - env.getAlterInstance().processAlterMaterializedView(alterView, true); break; } case OperationType.OP_ALTER_USER: { @@ -1852,26 +1823,6 @@ public class EditLog { logEdit(id, log); } - public void logCreateMTMVJob(MTMVJob job) { - logEdit(OperationType.OP_CREATE_MTMV_JOB, job); - } - - public void logDropMTMVJob(List jobIds) { - logEdit(OperationType.OP_DROP_MTMV_JOB, new DropMTMVJob(jobIds)); - } - - public void logChangeMTMVJob(ChangeMTMVJob changeJob) { - logEdit(OperationType.OP_CHANGE_MTMV_JOB, changeJob); - } - - public void logCreateMTMVTask(MTMVTask task) { - logEdit(OperationType.OP_CREATE_MTMV_TASK, task); - } - - public void logDropMTMVTasks(List taskIds) { - logEdit(OperationType.OP_DROP_MTMV_TASK, new DropMTMVTask(taskIds)); - } - public void logInitCatalog(InitCatalogLog log) { logEdit(OperationType.OP_INIT_CATALOG, log); } @@ -1941,10 +1892,6 @@ public class EditLog { logEdit(OperationType.OP_ALTER_USER, log); } - public void logAlterMTMV(AlterMultiMaterializedView log) { - logEdit(OperationType.OP_ALTER_MTMV_STMT, log); - } - public void logCleanQueryStats(CleanQueryStatsInfo log) { logEdit(OperationType.OP_CLEAN_QUERY_STATS, log); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java index db7d7a28a2..6484a9a041 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java @@ -271,16 +271,19 @@ public class OperationType { @Deprecated public static final short OP_INIT_EXTERNAL_TABLE = 329; - // scheduler job and task 330-350 + @Deprecated public static final short OP_CREATE_MTMV_JOB = 330; + @Deprecated public static final short OP_DROP_MTMV_JOB = 331; + @Deprecated public static final short OP_CHANGE_MTMV_JOB = 332; - + @Deprecated public static final short OP_CREATE_MTMV_TASK = 340; + @Deprecated public static final short OP_DROP_MTMV_TASK = 341; @Deprecated public static final short OP_CHANGE_MTMV_TASK = 342; - + @Deprecated public static final short OP_ALTER_MTMV_STMT = 345; public static final short OP_DROP_EXTERNAL_TABLE = 350; diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java index 885cdd0d89..b3425d7e8e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java @@ -197,12 +197,6 @@ public class MetaPersistMethod { metaPersistMethod.writeMethod = Env.class.getDeclaredMethod("saveCatalog", CountingDataOutputStream.class, long.class); break; - case "mtmvJobManager": - metaPersistMethod.readMethod = Env.class.getDeclaredMethod("loadMTMVJobManager", DataInputStream.class, - long.class); - metaPersistMethod.writeMethod = Env.class.getDeclaredMethod("saveMTMVJobManager", - CountingDataOutputStream.class, long.class); - break; case "globalFunction": metaPersistMethod.readMethod = Env.class.getDeclaredMethod("loadGlobalFunction", DataInputStream.class, long.class); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java index 99215bdd4b..caa63a7cd5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java @@ -38,7 +38,7 @@ public class PersistMetaModules { "masterInfo", "frontends", "backends", "datasource", "db", "alterJob", "recycleBin", "globalVariable", "cluster", "broker", "resources", "exportJob", "syncJob", "backupHandler", "paloAuth", "transactionState", "colocateTableIndex", "routineLoadJobs", "loadJobV2", "smallFiles", - "plugins", "deleteHandler", "sqlBlockRule", "policy", "mtmvJobManager", "globalFunction", "workloadGroups", + "plugins", "deleteHandler", "sqlBlockRule", "policy", "globalFunction", "workloadGroups", "binlogs", "resourceGroups", "AnalysisMgrV2", "AsyncJobManager", "JobTaskManager"); // Modules in this list is deprecated and will not be saved in meta file. (also should not be in MODULE_NAMES) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java index b09f531c07..483fda4ab5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java @@ -37,7 +37,6 @@ import org.apache.doris.analysis.AlterColumnStatsStmt; import org.apache.doris.analysis.AlterDatabasePropertyStmt; import org.apache.doris.analysis.AlterDatabaseQuotaStmt; import org.apache.doris.analysis.AlterDatabaseRename; -import org.apache.doris.analysis.AlterMaterializedViewStmt; import org.apache.doris.analysis.AlterPolicyStmt; import org.apache.doris.analysis.AlterResourceStmt; import org.apache.doris.analysis.AlterRoutineLoadStmt; @@ -64,7 +63,6 @@ import org.apache.doris.analysis.CreateFileStmt; import org.apache.doris.analysis.CreateFunctionStmt; import org.apache.doris.analysis.CreateJobStmt; import org.apache.doris.analysis.CreateMaterializedViewStmt; -import org.apache.doris.analysis.CreateMultiTableMaterializedViewStmt; import org.apache.doris.analysis.CreatePolicyStmt; import org.apache.doris.analysis.CreateRepositoryStmt; import org.apache.doris.analysis.CreateResourceStmt; @@ -106,7 +104,6 @@ import org.apache.doris.analysis.RecoverTableStmt; import org.apache.doris.analysis.RefreshCatalogStmt; import org.apache.doris.analysis.RefreshDbStmt; import org.apache.doris.analysis.RefreshLdapStmt; -import org.apache.doris.analysis.RefreshMaterializedViewStmt; import org.apache.doris.analysis.RefreshTableStmt; import org.apache.doris.analysis.RestoreStmt; import org.apache.doris.analysis.ResumeJobStmt; @@ -154,8 +151,6 @@ public class DdlExecutor { EncryptKeyHelper.createEncryptKey((CreateEncryptKeyStmt) ddlStmt); } else if (ddlStmt instanceof DropEncryptKeyStmt) { EncryptKeyHelper.dropEncryptKey((DropEncryptKeyStmt) ddlStmt); - } else if (ddlStmt instanceof CreateMultiTableMaterializedViewStmt) { - env.createMultiTableMaterializedView((CreateMultiTableMaterializedViewStmt) ddlStmt); } else if (ddlStmt instanceof CreateTableStmt) { env.createTable((CreateTableStmt) ddlStmt); } else if (ddlStmt instanceof CreateTableLikeStmt) { @@ -340,12 +335,8 @@ public class DdlExecutor { env.getCatalogMgr().alterCatalogProps((AlterCatalogPropertyStmt) ddlStmt); } else if (ddlStmt instanceof CleanLabelStmt) { env.getLoadManager().cleanLabel((CleanLabelStmt) ddlStmt); - } else if (ddlStmt instanceof AlterMaterializedViewStmt) { - env.alterMaterializedView((AlterMaterializedViewStmt) ddlStmt); } else if (ddlStmt instanceof DropMaterializedViewStmt) { env.dropMaterializedView((DropMaterializedViewStmt) ddlStmt); - } else if (ddlStmt instanceof RefreshMaterializedViewStmt) { - env.refreshMaterializedView((RefreshMaterializedViewStmt) ddlStmt); } else if (ddlStmt instanceof RefreshCatalogStmt) { env.getCatalogMgr().refreshCatalog((RefreshCatalogStmt) ddlStmt); } else if (ddlStmt instanceof RefreshLdapStmt) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 606a1a203f..8c72d26087 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -69,8 +69,6 @@ import org.apache.doris.analysis.ShowLastInsertStmt; import org.apache.doris.analysis.ShowLoadProfileStmt; import org.apache.doris.analysis.ShowLoadStmt; import org.apache.doris.analysis.ShowLoadWarningsStmt; -import org.apache.doris.analysis.ShowMTMVJobStmt; -import org.apache.doris.analysis.ShowMTMVTaskStmt; import org.apache.doris.analysis.ShowPartitionIdStmt; import org.apache.doris.analysis.ShowPartitionsStmt; import org.apache.doris.analysis.ShowPluginsStmt; @@ -190,9 +188,6 @@ import org.apache.doris.load.LoadJob; import org.apache.doris.load.LoadJob.JobState; import org.apache.doris.load.loadv2.LoadManager; import org.apache.doris.load.routineload.RoutineLoadJob; -import org.apache.doris.mtmv.MTMVJobManager; -import org.apache.doris.mtmv.metadata.MTMVJob; -import org.apache.doris.mtmv.metadata.MTMVTask; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.scheduler.job.Job; import org.apache.doris.scheduler.job.JobTask; @@ -425,10 +420,6 @@ public class ShowExecutor { handleCopyTablet(); } else if (stmt instanceof ShowCatalogRecycleBinStmt) { handleShowCatalogRecycleBin(); - } else if (stmt instanceof ShowMTMVJobStmt) { - handleMTMVJobs(); - } else if (stmt instanceof ShowMTMVTaskStmt) { - handleMTMVTasks(); } else if (stmt instanceof ShowTypeCastStmt) { handleShowTypeCastStmt(); } else if (stmt instanceof ShowBuildIndexStmt) { @@ -2796,46 +2787,6 @@ public class ShowExecutor { resultSet = new ShowResultSet(showStmt.getMetaData(), infos); } - private void handleMTMVJobs() throws AnalysisException { - ShowMTMVJobStmt showStmt = (ShowMTMVJobStmt) stmt; - MTMVJobManager jobManager = Env.getCurrentEnv().getMTMVJobManager(); - List jobs = Lists.newArrayList(); - if (showStmt.isShowAllJobs()) { - jobs.addAll(jobManager.showAllJobs()); - } else if (showStmt.isShowAllJobsFromDb()) { - jobs.addAll(jobManager.showJobs(showStmt.getDbName())); - } else if (showStmt.isShowAllJobsOnMv()) { - jobs.addAll(jobManager.showJobs(showStmt.getDbName(), showStmt.getMVName())); - } else if (showStmt.isSpecificJob()) { - jobs.add(jobManager.getJob(showStmt.getJobName())); - } - List> results = Lists.newArrayList(); - for (MTMVJob job : jobs) { - results.add(job.toStringRow()); - } - resultSet = new ShowResultSet(showStmt.getMetaData(), results); - } - - private void handleMTMVTasks() throws AnalysisException { - ShowMTMVTaskStmt showStmt = (ShowMTMVTaskStmt) stmt; - MTMVJobManager jobManager = Env.getCurrentEnv().getMTMVJobManager(); - List tasks = Lists.newArrayList(); - if (showStmt.isShowAllTasks()) { - tasks.addAll(jobManager.getTaskManager().showAllTasks()); - } else if (showStmt.isShowAllTasksFromDb()) { - tasks.addAll(jobManager.getTaskManager().showTasksWithLock(showStmt.getDbName())); - } else if (showStmt.isShowAllTasksOnMv()) { - tasks.addAll(jobManager.getTaskManager().showTasks(showStmt.getDbName(), showStmt.getMVName())); - } else if (showStmt.isSpecificTask()) { - tasks.add(jobManager.getTaskManager().getTask(showStmt.getTaskId())); - } - List> results = Lists.newArrayList(); - for (MTMVTask task : tasks) { - results.add(task.toStringRow()); - } - resultSet = new ShowResultSet(showStmt.getMetaData(), results); - } - private void handleShowTypeCastStmt() throws AnalysisException { ShowTypeCastStmt showStmt = (ShowTypeCastStmt) stmt; diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/MultiTableMaterializedViewTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/MultiTableMaterializedViewTest.java deleted file mode 100644 index 026406e94f..0000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/MultiTableMaterializedViewTest.java +++ /dev/null @@ -1,577 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.catalog; - -import org.apache.doris.analysis.CreateMultiTableMaterializedViewStmt; -import org.apache.doris.analysis.DropMaterializedViewStmt; -import org.apache.doris.analysis.DropTableStmt; -import org.apache.doris.analysis.MVRefreshInfo; -import org.apache.doris.analysis.MVRefreshIntervalTriggerInfo; -import org.apache.doris.analysis.ShowStmt; -import org.apache.doris.catalog.TableIf.TableType; -import org.apache.doris.common.Config; -import org.apache.doris.common.DdlException; -import org.apache.doris.common.ExceptionChecker; -import org.apache.doris.common.UserException; -import org.apache.doris.common.io.DataInputBuffer; -import org.apache.doris.common.io.DataOutputBuffer; -import org.apache.doris.common.util.SqlParserUtils; -import org.apache.doris.common.util.Util; -import org.apache.doris.qe.ShowExecutor; -import org.apache.doris.qe.ShowResultSet; -import org.apache.doris.qe.StmtExecutor; -import org.apache.doris.thrift.TStorageType; -import org.apache.doris.utframe.TestWithFeService; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - -import java.io.IOException; - -public class MultiTableMaterializedViewTest extends TestWithFeService { - - @BeforeEach - protected void setUp() throws Exception { - createDatabase("test"); - connectContext.setDatabase("default_cluster:test"); - connectContext.getState().reset(); - Config.enable_mtmv = true; - } - - @AfterEach - public void tearDown() { - Env.getCurrentEnv().clear(); - } - - @Test - public void testSerialization() throws Exception { - createTable("create table test.t1 (pk int, v1 int sum) aggregate key (pk) " - + "distributed by hash (pk) buckets 1 properties ('replication_num' = '1');"); - createTable("create table test.t2 (pk int, v2 int sum) aggregate key (pk) " - + "distributed by hash (pk) buckets 1 properties ('replication_num' = '1');"); - - String sql = "create materialized view mv build immediate refresh complete " - + "key (mpk) distributed by hash (mpk) " - + "as select test.t1.pk as mpk from test.t1, test.t2 where test.t1.pk = test.t2.pk"; - testSerialization(sql); - - sql = "create materialized view mv1 build immediate refresh complete start with '1:00' next 1 day " - + "key (mpk) distributed by hash (mpk) " - + "as select test.t1.pk as mpk from test.t1, test.t2 where test.t1.pk = test.t2.pk"; - testSerialization(sql); - } - - private void testSerialization(String sql) throws UserException, IOException { - MaterializedView mv = createMaterializedView(sql); - DataOutputBuffer out = new DataOutputBuffer(1024); - mv.write(out); - DataInputBuffer in = new DataInputBuffer(); - in.reset(out.getData(), out.getLength()); - MaterializedView other = new MaterializedView(); - other.readFields(in); - - Assertions.assertEquals(TableType.MATERIALIZED_VIEW, mv.getType()); - Assertions.assertEquals(mv.getType(), other.getType()); - Assertions.assertEquals(mv.getName(), other.getName()); - Assertions.assertEquals(mv.getQuery(), other.getQuery()); - - MVRefreshInfo refreshInfo = mv.getRefreshInfo(); - MVRefreshInfo otherRefreshInfo = other.getRefreshInfo(); - Assertions.assertEquals(refreshInfo.isNeverRefresh(), otherRefreshInfo.isNeverRefresh()); - Assertions.assertEquals(refreshInfo.getRefreshMethod(), otherRefreshInfo.getRefreshMethod()); - - Assertions.assertEquals( - refreshInfo.getTriggerInfo().getRefreshTrigger(), - otherRefreshInfo.getTriggerInfo().getRefreshTrigger() - ); - - MVRefreshIntervalTriggerInfo intervalTrigger = refreshInfo.getTriggerInfo().getIntervalTrigger(); - MVRefreshIntervalTriggerInfo otherIntervalTrigger = otherRefreshInfo.getTriggerInfo().getIntervalTrigger(); - if (intervalTrigger == null) { - Assertions.assertNull(otherIntervalTrigger); - } else { - Assertions.assertEquals(intervalTrigger.getStartTime(), otherIntervalTrigger.getStartTime()); - Assertions.assertEquals(intervalTrigger.getInterval(), otherIntervalTrigger.getInterval()); - Assertions.assertEquals(intervalTrigger.getTimeUnit(), otherIntervalTrigger.getTimeUnit()); - } - } - - private MaterializedView createMaterializedView(String sql) throws UserException { - CreateMultiTableMaterializedViewStmt stmt = (CreateMultiTableMaterializedViewStmt) SqlParserUtils - .parseAndAnalyzeStmt(sql, connectContext); - MaterializedView mv = (MaterializedView) new OlapTableFactory() - .init(OlapTableFactory.getTableType(stmt)) - .withTableId(0) - .withTableName(stmt.getMVName()) - .withKeysType(stmt.getKeysDesc().getKeysType()) - .withSchema(stmt.getColumns()) - .withPartitionInfo(new SinglePartitionInfo()) - .withDistributionInfo(stmt.getDistributionDesc().toDistributionInfo(stmt.getColumns())) - .withExtraParams(stmt) - .build(); - mv.setBaseIndexId(1); - mv.setIndexMeta( - 1, - stmt.getMVName(), - stmt.getColumns(), - 0, - Util.generateSchemaHash(), - Env.calcShortKeyColumnCount(stmt.getColumns(), stmt.getProperties(), true), - TStorageType.COLUMN, - stmt.getKeysDesc().getKeysType()); - return mv; - } - - @Test - void testShowCreateTables() throws Exception { - createTable("create table test.t1 (pk int, v1 int sum) aggregate key (pk) " - + "distributed by hash (pk) buckets 1 properties ('replication_num' = '1');"); - createTable("create table test.t2 (pk int, v2 int sum) aggregate key (pk) " - + "distributed by hash (pk) buckets 1 properties ('replication_num' = '1');"); - new StmtExecutor(connectContext, "create materialized view mv " - + "build immediate refresh complete key (mpk) distributed by hash (mpk) " - + "properties ('replication_num' = '1') " - + "as select test.t1.pk as mpk from test.t1, test.t2 where test.t1.pk = test.t2.pk").execute(); - Assertions.assertNull(connectContext.getState().getErrorCode(), connectContext.getState().getErrorMessage()); - - ShowExecutor showExecutor = new ShowExecutor(connectContext, - (ShowStmt) parseAndAnalyzeStmt("show create table mv")); - ShowResultSet resultSet = showExecutor.execute(); - String result = resultSet.getResultRows().get(0).get(1); - Assertions.assertTrue(result.contains("CREATE MATERIALIZED VIEW `mv`\n" - + "BUILD IMMEDIATE REFRESH COMPLETE ON DEMAND\n" - + "KEY(`mpk`)\n" - + "DISTRIBUTED BY HASH(`mpk`) BUCKETS 10")); - } - - @Test - void testDropMaterializedView() throws Exception { - createTable("create table test.t1 (pk int, v1 int sum) aggregate key (pk) " - + "distributed by hash (pk) buckets 1 properties ('replication_num' = '1');"); - createTable("create table test.t2 (pk int, v2 int sum) aggregate key (pk) " - + "distributed by hash (pk) buckets 1 properties ('replication_num' = '1');"); - new StmtExecutor(connectContext, "create materialized view mv " - + "build immediate refresh complete key (mpk) distributed by hash (mpk) " - + "properties ('replication_num' = '1') " - + "as select test.t1.pk as mpk from test.t1, test.t2 where test.t1.pk = test.t2.pk").execute(); - Assertions.assertNull(connectContext.getState().getErrorCode(), connectContext.getState().getErrorMessage()); - - ExceptionChecker.expectThrowsWithMsg(DdlException.class, "is not TABLE", - () -> Env.getCurrentInternalCatalog() - .dropTable((DropTableStmt) parseAndAnalyzeStmt("drop table mv"))); - - ExceptionChecker.expectThrowsNoException(() -> connectContext.getEnv().dropMaterializedView( - (DropMaterializedViewStmt) parseAndAnalyzeStmt("drop materialized view mv"))); - } - - @ParameterizedTest - @ValueSource(strings = {"AGGREGATE", "UNIQUE", "DUPLICATE"}) - public void testCreate(String keyType) throws Exception { - String aggregation = keyType.equals("AGGREGATE") ? "SUM" : ""; - createTable("CREATE TABLE test.t1 (" - + " pk INT," - + " v1 INT " + aggregation - + ") " + keyType + " KEY (pk)" - + "DISTRIBUTED BY HASH(pk) BUCKETS 1 PROPERTIES ('replication_num' = '1')"); - createTable("CREATE TABLE test.t2 (" - + " pk INT," - + " v2 INT " + aggregation - + ") " + keyType + " KEY (pk)" - + "DISTRIBUTED BY HASH(pk) BUCKETS 1 PROPERTIES ('replication_num' = '1')"); - new StmtExecutor(connectContext, "CREATE MATERIALIZED VIEW mv " - + "BUILD IMMEDIATE REFRESH COMPLETE KEY (pk) DISTRIBUTED BY HASH(pk) " - + "PROPERTIES ('replication_num' = '1') " - + "AS SELECT t1.pk, v1, v2 FROM test.t1, test.t2 WHERE test.t1.pk = test.t2.pk").execute(); - Assertions.assertNull(connectContext.getState().getErrorCode(), connectContext.getState().getErrorMessage()); - - connectContext.getEnv().dropMaterializedView( - (DropMaterializedViewStmt) parseAndAnalyzeStmt("DROP MATERIALIZED VIEW mv")); - new StmtExecutor(connectContext, "CREATE MATERIALIZED VIEW mv " - + "BUILD IMMEDIATE REFRESH COMPLETE DISTRIBUTED BY HASH(pk) " - + "PROPERTIES ('replication_num' = '1') " - + "AS SELECT t1.pk, v1, v2 FROM test.t1, test.t2 WHERE test.t1.pk = test.t2.pk").execute(); - Assertions.assertNull(connectContext.getState().getErrorCode(), connectContext.getState().getErrorMessage()); - } - - @ParameterizedTest - @ValueSource(strings = {"AGGREGATE", "UNIQUE", "DUPLICATE"}) - public void testCreateWithAliases(String keyType) throws Exception { - String aggregation = keyType.equals("AGGREGATE") ? "SUM" : ""; - createTable("CREATE TABLE test.t1 (" - + " pk INT," - + " v1 INT " + aggregation - + ") " + keyType + " KEY (pk)" - + "DISTRIBUTED BY HASH(pk) BUCKETS 1 PROPERTIES ('replication_num' = '1')"); - createTable("CREATE TABLE test.t2 (" - + " pk INT," - + " v2 INT " + aggregation - + ") " + keyType + " KEY (pk)" - + "DISTRIBUTED BY HASH(pk) BUCKETS 1 PROPERTIES ('replication_num' = '1')"); - new StmtExecutor(connectContext, "CREATE MATERIALIZED VIEW mv " - + "BUILD IMMEDIATE REFRESH COMPLETE KEY (mpk) DISTRIBUTED BY HASH(mpk) " - + "PROPERTIES ('replication_num' = '1') " - + "AS SELECT t1.pk AS mpk, v1, v2 FROM test.t1, test.t2 WHERE test.t1.pk = test.t2.pk").execute(); - Assertions.assertNull(connectContext.getState().getErrorCode(), connectContext.getState().getErrorMessage()); - - connectContext.getEnv().dropMaterializedView( - (DropMaterializedViewStmt) parseAndAnalyzeStmt("DROP MATERIALIZED VIEW mv")); - new StmtExecutor(connectContext, "CREATE MATERIALIZED VIEW mv " - + "BUILD IMMEDIATE REFRESH COMPLETE DISTRIBUTED BY HASH(mpk) " - + "PROPERTIES ('replication_num' = '1') " - + "AS SELECT t1.pk as mpk, v1, v2 FROM test.t1, test.t2 WHERE test.t1.pk = test.t2.pk").execute(); - Assertions.assertNull(connectContext.getState().getErrorCode(), connectContext.getState().getErrorMessage()); - } - - @ParameterizedTest - @ValueSource(strings = {"AGGREGATE", "UNIQUE", "DUPLICATE"}) - @Disabled - public void testCreateWithPartition(String keyType) throws Exception { - String aggregation = keyType.equals("AGGREGATE") ? "SUM" : ""; - createTable("CREATE TABLE test.t1 (" - + " pk INT NOT NULL," - + " v1 INT " + aggregation - + ") " + keyType + " KEY (pk)" - + "PARTITION BY RANGE(pk) (" - + " PARTITION p1 VALUES LESS THAN ('10')," - + " PARTITION p2 VALUES LESS THAN ('20')" - + ")" - + "DISTRIBUTED BY HASH(pk) BUCKETS 1 PROPERTIES ('replication_num' = '1')"); - createTable("CREATE TABLE test.t2 (" - + " pk INT NOT NULL," - + " v2 INT " + aggregation - + ") " + keyType + " KEY (pk)" - + "PARTITION BY LIST(pk) (" - + " PARTITION odd VALUES IN ('10', '30', '50', '70', '90')," - + " PARTITION even VALUES IN ('20', '40', '60', '80')" - + ")" - + "DISTRIBUTED BY HASH(pk) BUCKETS 1 PROPERTIES ('replication_num' = '1')"); - - new StmtExecutor(connectContext, "CREATE MATERIALIZED VIEW mv " - + "BUILD IMMEDIATE REFRESH COMPLETE KEY (pk) " - + "PARTITION BY (t1.pk)" - + "DISTRIBUTED BY HASH(pk) " - + "PROPERTIES ('replication_num' = '1') " - + "AS SELECT t1.pk, v1, v2 FROM test.t1, test.t2 WHERE test.t1.pk = test.t2.pk").execute(); - Assertions.assertNull(connectContext.getState().getErrorCode(), connectContext.getState().getErrorMessage()); - - connectContext.getEnv().dropMaterializedView( - (DropMaterializedViewStmt) parseAndAnalyzeStmt("DROP MATERIALIZED VIEW mv")); - new StmtExecutor(connectContext, "CREATE MATERIALIZED VIEW mv " - + "BUILD IMMEDIATE REFRESH COMPLETE KEY (pk) " - + "PARTITION BY (t2.pk)" - + "DISTRIBUTED BY HASH(pk) " - + "PROPERTIES ('replication_num' = '1') " - + "AS SELECT t2.pk, v1, v2 FROM test.t1, test.t2 WHERE test.t1.pk = test.t2.pk").execute(); - Assertions.assertNull(connectContext.getState().getErrorCode(), connectContext.getState().getErrorMessage()); - - connectContext.getEnv().dropMaterializedView( - (DropMaterializedViewStmt) parseAndAnalyzeStmt("DROP MATERIALIZED VIEW mv")); - - connectContext.getState().reset(); - new StmtExecutor(connectContext, "CREATE MATERIALIZED VIEW mv " - + "BUILD IMMEDIATE REFRESH COMPLETE KEY (pk) " - + "PARTITION BY (t1.pk)" - + "DISTRIBUTED BY HASH(pk) " - + "PROPERTIES ('replication_num' = '1') " - + "AS SELECT t2.pk, v1, v2 FROM test.t1, test.t2 WHERE test.t1.pk = test.t2.pk").execute(); - Assertions.assertTrue( - connectContext.getState().getErrorMessage().contains("Failed to map the partition column name")); - - connectContext.getState().reset(); - new StmtExecutor(connectContext, "CREATE MATERIALIZED VIEW mv " - + "BUILD IMMEDIATE REFRESH COMPLETE KEY (pk) " - + "PARTITION BY (v1)" - + "DISTRIBUTED BY HASH(pk) " - + "PROPERTIES ('replication_num' = '1') " - + "AS SELECT t2.pk, v1, v2 FROM test.t1, test.t2 WHERE test.t1.pk = test.t2.pk").execute(); - Assertions.assertTrue( - connectContext.getState().getErrorMessage() - .contains("The partition columns doesn't match the ones in base table")); - } - - @Test - public void testCreateWithTableAliases() throws Exception { - createTable("CREATE TABLE t_user (" - + " event_day DATE," - + " id bigint," - + " username varchar(20)" - + ")" - + "DISTRIBUTED BY HASH(id) BUCKETS 10 " - + "PROPERTIES ('replication_num' = '1')" - ); - createTable("CREATE TABLE t_user_pv(" - + " event_day DATE," - + " id bigint," - + " pv bigint" - + ")" - + "DISTRIBUTED BY HASH(id) BUCKETS 10 " - + "PROPERTIES ('replication_num' = '1')" - ); - new StmtExecutor(connectContext, "CREATE MATERIALIZED VIEW mv " - + "BUILD IMMEDIATE REFRESH COMPLETE " - + "START WITH \"2022-10-27 19:35:00\" " - + "NEXT 1 SECOND " - + "KEY (username) " - + "DISTRIBUTED BY HASH(username) BUCKETS 10 " - + "PROPERTIES ('replication_num' = '1') " - + "AS SELECT t1.username ,t2.pv FROM t_user t1 LEFT JOIN t_user_pv t2 on t1.id = t2.id").execute(); - Assertions.assertNull(connectContext.getState().getErrorCode(), connectContext.getState().getErrorMessage()); - } - - @Test - public void testCreateWithHint() throws Exception { - createTable("CREATE TABLE t_user (" - + " event_day DATE," - + " id bigint," - + " username varchar(20)" - + ")" - + "DISTRIBUTED BY HASH(id) BUCKETS 10 " - + "PROPERTIES ('replication_num' = '1')" - ); - createTable("CREATE TABLE t_user_pv(" - + " event_day DATE," - + " id bigint," - + " pv bigint" - + ")" - + "DISTRIBUTED BY HASH(id) BUCKETS 10 " - + "PROPERTIES ('replication_num' = '1')" - ); - new StmtExecutor(connectContext, "CREATE MATERIALIZED VIEW mv " - + "BUILD IMMEDIATE REFRESH COMPLETE " - + "DISTRIBUTED BY HASH(username) BUCKETS 10 " - + "PROPERTIES ('replication_num' = '1') " - + "AS SELECT /*+ SET_VAR(exec_mem_limit=1048576, query_timeout=3600) */ " - + "t1.username ,t2.pv FROM t_user t1 LEFT JOIN t_user_pv t2 on t1.id = t2.id").execute(); - Assertions.assertNull(connectContext.getState().getErrorCode(), connectContext.getState().getErrorMessage()); - ShowExecutor showExecutor = new ShowExecutor(connectContext, - (ShowStmt) parseAndAnalyzeStmt("show create table mv")); - ShowResultSet resultSet = showExecutor.execute(); - String result = resultSet.getResultRows().get(0).get(1); - Assertions.assertTrue( - result.contains("SELECT /*+ SET_VAR(exec_mem_limit=1048576, query_timeout=3600) */") - || result.contains("SELECT /*+ SET_VAR(query_timeout=3600, exec_mem_limit=1048576) */") - ); - } - - @Test - public void testCreateWithViews() throws Exception { - createTable("CREATE TABLE lineorder (" - + " lo_orderkey int," - + " lo_linenumber int," - + " lo_custkey int," - + " lo_partkey int," - + " lo_suppkey int," - + " lo_orderdate int," - + " lo_orderpriority int," - + " lo_shippriority int," - + " lo_quantity int," - + " lo_extendedprice int," - + " lo_ordtotalprice int," - + " lo_discount int," - + " lo_revenue int," - + " lo_supplycost int," - + " lo_tax int," - + " lo_commitdate int," - + " lo_shipmode int)" - + "DUPLICATE KEY (lo_orderkey)" - + "PARTITION BY RANGE (lo_orderdate) (" - + " PARTITION p1 VALUES [(\"-2147483648\"), (\"19930101\"))," - + " PARTITION p2 VALUES [(\"19930101\"), (\"19940101\"))," - + " PARTITION p3 VALUES [(\"19940101\"), (\"19950101\"))," - + " PARTITION p4 VALUES [(\"19950101\"), (\"19960101\"))," - + " PARTITION p5 VALUES [(\"19960101\"), (\"19970101\"))," - + " PARTITION p6 VALUES [(\"19970101\"), (\"19980101\"))," - + " PARTITION p7 VALUES [(\"19980101\"), (\"19990101\")))" - + "DISTRIBUTED BY HASH(lo_orderkey) BUCKETS 48 " - + "PROPERTIES ('replication_num' = '1')"); - - createTable("CREATE TABLE customer (" - + " c_custkey int," - + " c_name varchar," - + " c_address varchar," - + " c_city varchar," - + " c_nation varchar," - + " c_region varchar," - + " c_phone varchar," - + " c_mktsegment varchar)" - + "DUPLICATE KEY (c_custkey)" - + "DISTRIBUTED BY HASH (c_custkey) BUCKETS 12 " - + "PROPERTIES ('replication_num' = '1')"); - - createTable("CREATE TABLE supplier (" - + " s_suppkey int," - + " s_name varchar," - + " s_address varchar," - + " s_city varchar," - + " s_nation varchar," - + " s_region varchar," - + " s_phone varchar)" - + "DUPLICATE KEY (s_suppkey)" - + "DISTRIBUTED BY HASH (s_suppkey) BUCKETS 12 " - + "PROPERTIES ('replication_num' = '1')"); - - createTable("CREATE TABLE part (" - + " p_partkey int," - + " p_name varchar," - + " p_mfgr varchar," - + " p_category varchar," - + " p_brand varchar," - + " p_color varchar," - + " p_type varchar," - + " p_size int," - + " p_container varchar)" - + "DUPLICATE KEY (p_partkey)" - + "DISTRIBUTED BY HASH (p_partkey) BUCKETS 12 " - + "PROPERTIES ('replication_num' = '1')"); - - new StmtExecutor(connectContext, "CREATE MATERIALIZED VIEW ssb_view " - + "BUILD IMMEDIATE REFRESH COMPLETE " - + "DISTRIBUTED BY HASH (LO_ORDERKEY) " - + "PROPERTIES ('replication_num' = '1') " - + "AS SELECT " - + " LO_ORDERDATE," - + " LO_ORDERKEY," - + " LO_LINENUMBER," - + " LO_CUSTKEY," - + " LO_PARTKEY," - + " LO_SUPPKEY," - + " LO_ORDERPRIORITY," - + " LO_SHIPPRIORITY," - + " LO_QUANTITY," - + " LO_EXTENDEDPRICE," - + " LO_ORDTOTALPRICE," - + " LO_DISCOUNT," - + " LO_REVENUE," - + " LO_SUPPLYCOST," - + " LO_TAX," - + " LO_COMMITDATE," - + " LO_SHIPMODE," - + " C_NAME," - + " C_ADDRESS," - + " C_CITY," - + " C_NATION," - + " C_REGION," - + " C_PHONE," - + " C_MKTSEGMENT," - + " S_NAME," - + " S_ADDRESS," - + " S_CITY," - + " S_NATION," - + " S_REGION," - + " S_PHONE," - + " P_NAME," - + " P_MFGR," - + " P_CATEGORY," - + " P_BRAND," - + " P_COLOR," - + " P_TYPE," - + " P_SIZE," - + " P_CONTAINER " - + "FROM (" - + " SELECT " - + " lo_orderkey," - + " lo_linenumber," - + " lo_custkey," - + " lo_partkey," - + " lo_suppkey," - + " lo_orderdate," - + " lo_orderpriority," - + " lo_shippriority," - + " lo_quantity," - + " lo_extendedprice," - + " lo_ordtotalprice," - + " lo_discount," - + " lo_revenue," - + " lo_supplycost," - + " lo_tax," - + " lo_commitdate," - + " lo_shipmode" - + " FROM lineorder" - + " WHERE lo_orderdate<19930101" - + ") l " - + "INNER JOIN customer c ON (c.c_custkey = l.lo_custkey) " - + "INNER JOIN supplier s ON (s.s_suppkey = l.lo_suppkey) " - + "INNER JOIN part p ON (p.p_partkey = l.lo_partkey)").execute(); - Assertions.assertNull(connectContext.getState().getErrorCode(), connectContext.getState().getErrorMessage()); - } - - @Test - void testCreateNeverRefreshMaterializedView() throws Exception { - createTable("create table test.t1 (pk int, v1 int sum) aggregate key (pk) " - + "distributed by hash (pk) buckets 1 properties ('replication_num' = '1');"); - createTable("create table test.t2 (pk int, v2 int sum) aggregate key (pk) " - + "distributed by hash (pk) buckets 1 properties ('replication_num' = '1');"); - new StmtExecutor(connectContext, "create materialized view mv " - + "build immediate never refresh key (mpk) distributed by hash (mpk) " - + "properties ('replication_num' = '1') " - + "as select test.t1.pk as mpk from test.t1, test.t2 where test.t1.pk = test.t2.pk").execute(); - Assertions.assertNull(connectContext.getState().getErrorCode(), connectContext.getState().getErrorMessage()); - - ShowExecutor showExecutor = new ShowExecutor(connectContext, - (ShowStmt) parseAndAnalyzeStmt("show create table mv")); - ShowResultSet resultSet = showExecutor.execute(); - String result = resultSet.getResultRows().get(0).get(1); - Assertions.assertTrue(result.contains("CREATE MATERIALIZED VIEW `mv`\n" - + "BUILD IMMEDIATE NEVER REFRESH \n" - + "KEY(`mpk`)\n" - + "DISTRIBUTED BY HASH(`mpk`) BUCKETS 10")); - } - - @Test - void testCreateWithStar() throws Exception { - createTable("CREATE TABLE t_user (" - + " event_day DATE," - + " id bigint," - + " username varchar(20)" - + ")" - + "DISTRIBUTED BY HASH(id) BUCKETS 10 " - + "PROPERTIES ('replication_num' = '1')" - ); - new StmtExecutor(connectContext, "CREATE MATERIALIZED VIEW mv " - + "BUILD IMMEDIATE REFRESH COMPLETE " - + "START WITH \"2022-10-27 19:35:00\" " - + "NEXT 1 SECOND " - + "DISTRIBUTED BY HASH(username) BUCKETS 10 " - + "PROPERTIES ('replication_num' = '1') " - + "AS SELECT t1.* FROM t_user t1").execute(); - Assertions.assertNull(connectContext.getState().getErrorCode(), connectContext.getState().getErrorMessage()); - } - - @Test - void testCreateWithoutRefreshInfo() throws Exception { - createTable("CREATE TABLE t_user (" - + " event_day DATE," - + " id bigint," - + " username varchar(20)" - + ")" - + "DISTRIBUTED BY HASH(id) BUCKETS 10 " - + "PROPERTIES ('replication_num' = '1')" - ); - new StmtExecutor(connectContext, "CREATE MATERIALIZED VIEW mv " - + "BUILD DEFERRED " - + "DISTRIBUTED BY HASH(username) BUCKETS 10 " - + "PROPERTIES ('replication_num' = '1') " - + "AS SELECT t1.* FROM t_user t1").execute(); - Assertions.assertNull(connectContext.getState().getErrorCode(), connectContext.getState().getErrorMessage()); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVJobManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVJobManagerTest.java deleted file mode 100644 index 704551c1d2..0000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVJobManagerTest.java +++ /dev/null @@ -1,145 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.mtmv; - -import org.apache.doris.catalog.Env; -import org.apache.doris.common.DdlException; -import org.apache.doris.metric.MetricRepo; -import org.apache.doris.mtmv.MTMVUtils.JobState; -import org.apache.doris.mtmv.metadata.ChangeMTMVJob; -import org.apache.doris.mtmv.metadata.MTMVJob; -import org.apache.doris.mtmv.metadata.MTMVTask; -import org.apache.doris.utframe.TestWithFeService; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.Collections; -import java.util.List; - -public class MTMVJobManagerTest extends TestWithFeService { - - @Test - public void testSampleCase() throws DdlException { - String jobName = "testSampleCaseJob"; - String mvName = "testSampleCaseMv"; - MTMVJobManager jobManager = Env.getCurrentEnv().getMTMVJobManager(); - MTMVJob job = MTMVUtilsTest.createDummyJob(mvName, jobName); - jobManager.createJob(job, false); - Assertions.assertNotNull(jobManager.getJob(job.getName())); - MTMVJob resultJob = jobManager.getJob(jobName); - Assertions.assertEquals(JobState.ACTIVE, resultJob.getState()); - long jobId = resultJob.getId(); - ChangeMTMVJob changeMTMVJob = new ChangeMTMVJob(jobId, JobState.PAUSE); - resultJob.updateJob(changeMTMVJob, false); - resultJob = jobManager.getJob(jobName); - Assertions.assertEquals(jobName, resultJob.getName()); - Assertions.assertEquals(JobState.PAUSE, resultJob.getState()); - jobManager.dropJobs(Collections.singletonList(jobId), false); - Assertions.assertNull(jobManager.getJob(jobName)); - } - - @Test - public void testSchedulerJob() throws DdlException, InterruptedException { - String jobName = "testSchedulerJob"; - String mvName = "testSchedulerJobMv"; - MTMVJobManager jobManager = Env.getCurrentEnv().getMTMVJobManager(); - MTMVJob job = MTMVUtilsTest.createSchedulerJob(mvName, jobName); - jobManager.createJob(job, false); - Assertions.assertNotNull(jobManager.getJob(jobName)); - while (jobManager.getTaskManager().getHistoryTasksByJobName(jobName).isEmpty()) { - Thread.sleep(1000L); - System.out.println("Loop once"); - } - Assertions.assertTrue(jobManager.getTaskManager().getHistoryTasksByJobName(jobName).size() > 0); - } - - @Test - public void testOnceJob() throws DdlException, InterruptedException { - String jobName = "testOnceJob"; - String mvName = "testOnceJobMv"; - MTMVJobManager jobManager = Env.getCurrentEnv().getMTMVJobManager(); - MTMVJob job = MTMVUtilsTest.createOnceJob(mvName, jobName); - jobManager.createJob(job, false); - Assertions.assertNotNull(jobManager.getJob(jobName)); - while (!jobManager.getJob(jobName).getState().equals(JobState.COMPLETE)) { - Thread.sleep(1000L); - System.out.println("Loop once"); - } - - Assertions.assertEquals(1, jobManager.getTaskManager().getHistoryTasksByJobName(jobName).size()); - Assertions.assertEquals(1, - jobManager.getTaskManager().showTasks(MTMVUtilsTest.dbName, mvName).size()); - - // verify job meta - MTMVJob metaJob = jobManager.getJob(jobName); - List jobRow = metaJob.toStringRow(); - Assertions.assertEquals(13, jobRow.size()); - // index 1: Name - Assertions.assertEquals(jobName, jobRow.get(1)); - // index 2: TriggerMode - Assertions.assertEquals("ONCE", jobRow.get(2)); - // index 3: Schedule - Assertions.assertEquals("NULL", jobRow.get(3)); - // index 4: DBName - Assertions.assertEquals(MTMVUtilsTest.dbName, jobRow.get(4)); - // index 5: MVName - Assertions.assertEquals(mvName, jobRow.get(5)); - // index 6: Query - Assertions.assertEquals("", jobRow.get(6)); - // index 7: User - Assertions.assertEquals("root", jobRow.get(7)); - // index 8: RetryPolicy - Assertions.assertEquals("NEVER", jobRow.get(8)); - // index 9: State - Assertions.assertEquals("COMPLETE", jobRow.get(9)); - - // verify task meta - MTMVTask metaTask = jobManager.getTaskManager().getHistoryTasksByJobName(jobName).get(0); - List taskRow = metaTask.toStringRow(); - Assertions.assertEquals(14, taskRow.size()); - // index 1: JobName - Assertions.assertEquals(jobName, taskRow.get(1)); - // index 2: DBName - Assertions.assertEquals(MTMVUtilsTest.dbName, taskRow.get(2)); - // index 3: MVName - Assertions.assertEquals(mvName, taskRow.get(3)); - // index 4: Query - Assertions.assertEquals("", taskRow.get(4)); - // index 5: User - Assertions.assertEquals("root", taskRow.get(5)); - // index 6: Priority - Assertions.assertEquals("0", taskRow.get(6)); - // index 7: RetryTimes - Assertions.assertEquals("0", taskRow.get(7)); - // index 8: State - Assertions.assertEquals("FAILURE", taskRow.get(8)); - // index 9: Message - Assertions.assertEquals("", taskRow.get(9)); - // index 10: ErrorCode - //Assertions.assertEquals("0", taskRow.get(10)); - } - - @Test - public void testMetrics() { - int jobMetricCount = MetricRepo.DORIS_METRIC_REGISTER.getMetricsByName("mtmv_job").size(); - int taskMetricCount = MetricRepo.DORIS_METRIC_REGISTER.getMetricsByName("mtmv_task").size(); - Assertions.assertEquals(2, jobMetricCount); - Assertions.assertEquals(4, taskMetricCount); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskExecutorTest.java deleted file mode 100644 index 7bd58b145a..0000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskExecutorTest.java +++ /dev/null @@ -1,116 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.mtmv; - -import org.apache.doris.mtmv.MTMVUtils.TaskRetryPolicy; -import org.apache.doris.mtmv.MTMVUtils.TaskState; -import org.apache.doris.mtmv.metadata.MTMVJob; -import org.apache.doris.utframe.TestWithFeService; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import java.util.UUID; -import java.util.concurrent.ExecutionException; - -public class MTMVTaskExecutorTest extends TestWithFeService { - @Test - public void testSubmitTask() throws InterruptedException, ExecutionException { - String mvName = "testSubmitTaskMv"; - String jobName = "testSubmitTaskJob"; - MTMVTaskExecutorPool pool = new MTMVTaskExecutorPool(); - MTMVTaskExecutor executor = new MTMVTaskExecutor(); - executor.setProcessor(new MTMVTaskProcessor()); - executor.setJob(MTMVUtilsTest.createDummyJob(mvName, jobName)); - executor.initTask(UUID.randomUUID().toString(), System.currentTimeMillis()); - pool.executeTask(executor); - executor.getFuture().get(); - Assertions.assertEquals(TaskState.FAILURE, executor.getTask().getState()); - } - - - @Test - public void testFailTask() throws InterruptedException, ExecutionException { - String mvName = "testFailTaskMv"; - String jobName = "testFailTaskJob"; - MTMVTaskExecutorPool pool = new MTMVTaskExecutorPool(); - MTMVTaskExecutor executor = new MTMVTaskExecutor(); - executor.setProcessor(new MTMVTaskProcessorTest(1)); - executor.setJob(MTMVUtilsTest.createDummyJob(mvName, jobName)); - executor.initTask(UUID.randomUUID().toString(), System.currentTimeMillis()); - pool.executeTask(executor); - executor.getFuture().get(); - Assertions.assertEquals(TaskState.FAILURE, executor.getTask().getState()); - //Assertions.assertEquals("java.lang.Exception: my define error 1", executor.getTask().getMessage()); - } - - @Test - @Disabled - public void testRetryTask() throws InterruptedException, ExecutionException { - String mvName = "testRetryTaskMv"; - String jobName = "testRetryTaskJob"; - MTMVTaskExecutorPool pool = new MTMVTaskExecutorPool(); - - MTMVTaskExecutor executor = new MTMVTaskExecutor(); - executor.setProcessor(new MTMVTaskProcessorTest(3)); - MTMVJob job = MTMVUtilsTest.createDummyJob(mvName, jobName); - job.setRetryPolicy(TaskRetryPolicy.TIMES); - executor.setJob(job); - executor.initTask(UUID.randomUUID().toString(), System.currentTimeMillis()); - pool.executeTask(executor); - executor.getFuture().get(); - Assertions.assertEquals(TaskState.FAILURE, executor.getTask().getState()); - } - - @Test - @Disabled - public void testRetryFailTask() throws InterruptedException, ExecutionException { - String mvName = "testRetryTaskMv"; - String jobName = "testRetryTaskJob"; - MTMVTaskExecutorPool pool = new MTMVTaskExecutorPool(); - - MTMVTaskExecutor executor = new MTMVTaskExecutor(); - executor.setProcessor(new MTMVTaskProcessorTest(4)); - MTMVJob job = MTMVUtilsTest.createDummyJob(mvName, jobName); - job.setRetryPolicy(TaskRetryPolicy.TIMES); - executor.setJob(job); - executor.initTask(UUID.randomUUID().toString(), System.currentTimeMillis()); - pool.executeTask(executor); - executor.getFuture().get(); - Assertions.assertEquals(TaskState.FAILURE, executor.getTask().getState()); - //Assertions.assertEquals("java.lang.Exception: my define error 4", executor.getTask().getMessage()); - } - - public static class MTMVTaskProcessorTest extends MTMVTaskProcessor { - private final int times; - private int runTimes = 0; - - public MTMVTaskProcessorTest(int times) { - this.times = times; - } - - boolean process(MTMVTaskContext context) throws Exception { - if (runTimes < times) { - runTimes++; - throw new Exception("my define error " + runTimes); - } - return true; - } - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVUtilsTest.java deleted file mode 100644 index e76e26ec0d..0000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVUtilsTest.java +++ /dev/null @@ -1,90 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.mtmv; - -import org.apache.doris.mtmv.MTMVUtils.TriggerMode; -import org.apache.doris.mtmv.metadata.MTMVJob; -import org.apache.doris.mtmv.metadata.MTMVJob.JobSchedule; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.concurrent.TimeUnit; - -public class MTMVUtilsTest { - public static final String dbName = "test"; - - public static final String MV_NAME = "mvName"; - - public static MTMVJob createDummyJob(String mvName, String jobName) { - MTMVJob job = new MTMVJob(jobName); - job.setDBName(dbName); - job.setMVName(mvName); - return job; - } - - public static MTMVJob createOnceJob(String mvName, String jobName) { - MTMVJob job = new MTMVJob(jobName); - job.setTriggerMode(TriggerMode.ONCE); - job.setDBName(dbName); - job.setMVName(mvName); - return job; - } - - public static MTMVJob createSchedulerJob(String mvName, String jobName) { - MTMVJob job = new MTMVJob(jobName); - JobSchedule jobSchedule = new JobSchedule(System.currentTimeMillis() / 1000, 1, TimeUnit.SECONDS); - job.setSchedule(jobSchedule); - job.setTriggerMode(TriggerMode.PERIODICAL); - job.setDBName(dbName); - job.setMVName(mvName); - return job; - } - - @Test - public void testGetDelaySeconds() { - MTMVJob job = MTMVUtilsTest.createDummyJob("testGetDelaySecondsMv", "testGetDelaySecondsJob"); - - // 2022-10-03 15:00:00 - JobSchedule jobSchedule = new JobSchedule(1664780400L, 1, TimeUnit.HOURS); - job.setSchedule(jobSchedule); - - Assertions.assertEquals(0, MTMVUtils.getDelaySeconds(job)); - - // 2222-10-03 15:00:00 - jobSchedule = new JobSchedule(7976127600L, 1, TimeUnit.HOURS); - job.setSchedule(jobSchedule); - - // 2022-10-03 16:00:00 - LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochSecond(1664784000L), ZoneId.systemDefault()); - Assertions.assertEquals(7976127600L - 1664784000L, MTMVUtils.getDelaySeconds(job, time)); - - // 2022-10-03 15:00:00 - jobSchedule = new JobSchedule(1664780400L, 1, TimeUnit.HOURS); - job.setSchedule(jobSchedule); - // 2222-10-03 15:00:00 - job.setLastModifyTime(1664780400L + 1L); - Assertions.assertEquals(1L, MTMVUtils.getDelaySeconds(job, time)); - - job.setLastModifyTime(1664780400L - 1L); - Assertions.assertEquals(0L, MTMVUtils.getDelaySeconds(job, time)); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/ShowMTMVJobStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/ShowMTMVJobStmtTest.java deleted file mode 100644 index 99ae9abd78..0000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/ShowMTMVJobStmtTest.java +++ /dev/null @@ -1,103 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.mtmv; - -import org.apache.doris.analysis.AccessTestUtil; -import org.apache.doris.analysis.Analyzer; -import org.apache.doris.analysis.ShowMTMVJobStmt; -import org.apache.doris.analysis.TableName; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.UserException; - -import com.google.common.base.Strings; -import org.junit.Assert; -import org.junit.Test; - -public class ShowMTMVJobStmtTest { - @Test - public void testNormal() throws UserException, AnalysisException { - final Analyzer analyzer = AccessTestUtil.fetchEmptyDbAnalyzer(); - ShowMTMVJobStmt stmt = new ShowMTMVJobStmt(); - stmt.analyze(analyzer); - Assert.assertNull(stmt.getJobName()); - Assert.assertTrue(Strings.isNullOrEmpty(stmt.getDbName())); - Assert.assertNull(stmt.getMVName()); - Assert.assertEquals(13, stmt.getMetaData().getColumnCount()); - Assert.assertEquals("SHOW MTMV JOB", stmt.toSql()); - - stmt = new ShowMTMVJobStmt("job1"); - stmt.analyze(analyzer); - Assert.assertNotNull(stmt.getJobName()); - Assert.assertTrue(Strings.isNullOrEmpty(stmt.getDbName())); - Assert.assertNull(stmt.getMVName()); - Assert.assertEquals("SHOW MTMV JOB FOR job1", stmt.toSql()); - - stmt = new ShowMTMVJobStmt("db1", null); - stmt.analyze(analyzer); - Assert.assertNull(stmt.getJobName()); - Assert.assertFalse(Strings.isNullOrEmpty(stmt.getDbName())); - Assert.assertNull(stmt.getMVName()); - Assert.assertEquals("SHOW MTMV JOB FROM db1", stmt.toSql()); - - TableName tableName = new TableName(null, null, "mv1"); - stmt = new ShowMTMVJobStmt(null, tableName); - stmt.analyze(analyzer); - Assert.assertNull(stmt.getJobName()); - Assert.assertTrue(Strings.isNullOrEmpty(stmt.getDbName())); - Assert.assertNotNull(stmt.getMVName()); - Assert.assertEquals("SHOW MTMV JOB ON `mv1`", stmt.toSql()); - - tableName = new TableName(null, "db2", "mv1"); - stmt = new ShowMTMVJobStmt(null, tableName); - stmt.analyze(analyzer); - Assert.assertNull(stmt.getJobName()); - Assert.assertFalse(Strings.isNullOrEmpty(stmt.getDbName())); - Assert.assertNotNull(stmt.getMVName()); - Assert.assertEquals("SHOW MTMV JOB ON `db2`.`mv1`", stmt.toSql()); - - tableName = new TableName(null, null, "mv1"); - stmt = new ShowMTMVJobStmt("db1", tableName); - stmt.analyze(analyzer); - Assert.assertNull(stmt.getJobName()); - Assert.assertFalse(Strings.isNullOrEmpty(stmt.getDbName())); - Assert.assertNotNull(stmt.getMVName()); - Assert.assertEquals("SHOW MTMV JOB FROM db1 ON `mv1`", stmt.toSql()); - } - - @Test(expected = UserException.class) - public void testConflictDbName() throws UserException, AnalysisException { - final Analyzer analyzer = AccessTestUtil.fetchBlockAnalyzer(); - TableName tableName = new TableName(null, "db2", "mv1"); - ShowMTMVJobStmt stmt = new ShowMTMVJobStmt("db1", tableName); - stmt.analyze(analyzer); - } - - @Test - public void testDefaultDb() throws UserException { - final Analyzer analyzer = AccessTestUtil.fetchBlockAnalyzer(); - ShowMTMVJobStmt stmt = new ShowMTMVJobStmt(); - stmt.analyze(analyzer); - Assert.assertNull(stmt.getJobName()); - Assert.assertEquals("testCluster:testDb", stmt.getDbName()); - Assert.assertNull(stmt.getMVName()); - Assert.assertEquals(13, stmt.getMetaData().getColumnCount()); - Assert.assertEquals("SHOW MTMV JOB", stmt.toSql()); - Assert.assertFalse(stmt.isShowAllJobs()); - Assert.assertTrue(stmt.isShowAllJobsFromDb()); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/ShowMTMVTaskStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/ShowMTMVTaskStmtTest.java deleted file mode 100644 index 4c0d1f4d91..0000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/ShowMTMVTaskStmtTest.java +++ /dev/null @@ -1,104 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.mtmv; - -import org.apache.doris.analysis.AccessTestUtil; -import org.apache.doris.analysis.Analyzer; -import org.apache.doris.analysis.ShowMTMVTaskStmt; -import org.apache.doris.analysis.TableName; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.UserException; - -import com.google.common.base.Strings; -import org.junit.Assert; -import org.junit.Test; - -public class ShowMTMVTaskStmtTest { - @Test - public void testNormal() throws UserException, AnalysisException { - final Analyzer analyzer = AccessTestUtil.fetchEmptyDbAnalyzer(); - ShowMTMVTaskStmt stmt = new ShowMTMVTaskStmt(); - stmt.analyze(analyzer); - Assert.assertNull(stmt.getTaskId()); - Assert.assertTrue(Strings.isNullOrEmpty(stmt.getDbName())); - Assert.assertNull(stmt.getMVName()); - Assert.assertEquals(14, stmt.getMetaData().getColumnCount()); - Assert.assertEquals("SHOW MTMV TASK", stmt.toSql()); - - stmt = new ShowMTMVTaskStmt("task1"); - stmt.analyze(analyzer); - Assert.assertNotNull(stmt.getTaskId()); - Assert.assertTrue(Strings.isNullOrEmpty(stmt.getDbName())); - Assert.assertNull(stmt.getMVName()); - Assert.assertEquals("SHOW MTMV TASK FOR task1", stmt.toSql()); - - stmt = new ShowMTMVTaskStmt("db1", null); - stmt.analyze(analyzer); - Assert.assertNull(stmt.getTaskId()); - Assert.assertFalse(Strings.isNullOrEmpty(stmt.getDbName())); - Assert.assertNull(stmt.getMVName()); - Assert.assertEquals("SHOW MTMV TASK FROM db1", stmt.toSql()); - - TableName tableName = new TableName(null, null, "mv1"); - stmt = new ShowMTMVTaskStmt(null, tableName); - stmt.analyze(analyzer); - Assert.assertNull(stmt.getTaskId()); - Assert.assertTrue(Strings.isNullOrEmpty(stmt.getDbName())); - Assert.assertNotNull(stmt.getMVName()); - Assert.assertEquals("SHOW MTMV TASK ON `mv1`", stmt.toSql()); - - tableName = new TableName(null, "db2", "mv1"); - stmt = new ShowMTMVTaskStmt(null, tableName); - stmt.analyze(analyzer); - Assert.assertNull(stmt.getTaskId()); - Assert.assertFalse(Strings.isNullOrEmpty(stmt.getDbName())); - Assert.assertNotNull(stmt.getMVName()); - Assert.assertEquals("SHOW MTMV TASK ON `db2`.`mv1`", stmt.toSql()); - - tableName = new TableName(null, null, "mv1"); - stmt = new ShowMTMVTaskStmt("db1", tableName); - stmt.analyze(analyzer); - Assert.assertNull(stmt.getTaskId()); - Assert.assertFalse(Strings.isNullOrEmpty(stmt.getDbName())); - Assert.assertNotNull(stmt.getMVName()); - Assert.assertEquals("SHOW MTMV TASK FROM db1 ON `mv1`", stmt.toSql()); - } - - @Test(expected = UserException.class) - public void testConflictDbName() throws UserException, AnalysisException { - final Analyzer analyzer = AccessTestUtil.fetchBlockAnalyzer(); - TableName tableName = new TableName(null, "db2", "mv1"); - - ShowMTMVTaskStmt stmt = new ShowMTMVTaskStmt("db1", tableName); - stmt.analyze(analyzer); - } - - @Test - public void testDefaultDb() throws UserException { - final Analyzer analyzer = AccessTestUtil.fetchBlockAnalyzer(); - ShowMTMVTaskStmt stmt = new ShowMTMVTaskStmt(); - stmt.analyze(analyzer); - Assert.assertNull(stmt.getTaskId()); - Assert.assertEquals("testCluster:testDb", stmt.getDbName()); - Assert.assertNull(stmt.getMVName()); - Assert.assertEquals(14, stmt.getMetaData().getColumnCount()); - Assert.assertEquals("SHOW MTMV TASK", stmt.toSql()); - Assert.assertFalse(stmt.isShowAllTasks()); - Assert.assertTrue(stmt.isShowAllTasksFromDb()); - } -} diff --git a/regression-test/data/mtmv_p0/test_create_mtmv.out b/regression-test/data/mtmv_p0/test_create_mtmv.out deleted file mode 100644 index 75d5531799..0000000000 --- a/regression-test/data/mtmv_p0/test_create_mtmv.out +++ /dev/null @@ -1,6 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !select -- -clz 200 -lisi 300 -zhangsang 200 - diff --git a/regression-test/data/mtmv_p0/test_refresh_mtmv.out b/regression-test/data/mtmv_p0/test_refresh_mtmv.out deleted file mode 100644 index 75d5531799..0000000000 --- a/regression-test/data/mtmv_p0/test_refresh_mtmv.out +++ /dev/null @@ -1,6 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !select -- -clz 200 -lisi 300 -zhangsang 200 - diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 11e41e9bcf..4f56c75b82 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -528,29 +528,6 @@ class Suite implements GroovyInterceptable { file.delete() } - void waitingMTMVTaskFinished(String mvName) { - String showTasks = "SHOW MTMV TASK ON " + mvName - List> showTaskMetaResult = sql_meta(showTasks) - int index = showTaskMetaResult.indexOf(['State', 'CHAR']) - String status = "PENDING" - List> result - long startTime = System.currentTimeMillis() - long timeoutTimestamp = startTime + 30 * 60 * 1000 // 30 min - do { - result = sql(showTasks) - if (!result.isEmpty()) { - status = result.last().get(index) - } - println "The state of ${showTasks} is ${status}" - Thread.sleep(1000); - } while (timeoutTimestamp > System.currentTimeMillis() && (status == 'PENDING' || status == 'RUNNING')) - if (status != "SUCCESS") { - println "status is not success" - println result.toString() - } - Assert.assertEquals("SUCCESS", status) - } - List downloadExportFromHdfs(String label) { String dataDir = context.config.dataPath + "/" + group + "/" String hdfsFs = context.config.otherConfigs.get("hdfsFs") diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/customer_create.sql b/regression-test/suites/mtmv_p0/ssb/ddl/customer_create.sql deleted file mode 100644 index d13101b70e..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/customer_create.sql +++ /dev/null @@ -1,15 +0,0 @@ -CREATE TABLE IF NOT EXISTS `customer` ( - `c_custkey` int(11) NOT NULL COMMENT "", - `c_name` varchar(26) NOT NULL COMMENT "", - `c_address` varchar(41) NOT NULL COMMENT "", - `c_city` varchar(11) NOT NULL COMMENT "", - `c_nation` varchar(16) NOT NULL COMMENT "", - `c_region` varchar(13) NOT NULL COMMENT "", - `c_phone` varchar(16) NOT NULL COMMENT "", - `c_mktsegment` varchar(11) NOT NULL COMMENT "" -) -UNIQUE KEY (`c_custkey`) -DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 1 -PROPERTIES ( -"replication_num" = "1" -); \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/customer_delete.sql b/regression-test/suites/mtmv_p0/ssb/ddl/customer_delete.sql deleted file mode 100644 index fe22a226fe..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/customer_delete.sql +++ /dev/null @@ -1 +0,0 @@ -truncate table customer; \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/date_create.sql b/regression-test/suites/mtmv_p0/ssb/ddl/date_create.sql deleted file mode 100644 index 390905ebea..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/date_create.sql +++ /dev/null @@ -1,24 +0,0 @@ -CREATE TABLE IF NOT EXISTS `dates` ( - `d_datekey` int(11) NOT NULL COMMENT "", - `d_date` varchar(20) NOT NULL COMMENT "", - `d_dayofweek` varchar(10) NOT NULL COMMENT "", - `d_month` varchar(11) NOT NULL COMMENT "", - `d_year` int(11) NOT NULL COMMENT "", - `d_yearmonthnum` int(11) NOT NULL COMMENT "", - `d_yearmonth` varchar(9) NOT NULL COMMENT "", - `d_daynuminweek` int(11) NOT NULL COMMENT "", - `d_daynuminmonth` int(11) NOT NULL COMMENT "", - `d_daynuminyear` int(11) NOT NULL COMMENT "", - `d_monthnuminyear` int(11) NOT NULL COMMENT "", - `d_weeknuminyear` int(11) NOT NULL COMMENT "", - `d_sellingseason` varchar(14) NOT NULL COMMENT "", - `d_lastdayinweekfl` int(11) NOT NULL COMMENT "", - `d_lastdayinmonthfl` int(11) NOT NULL COMMENT "", - `d_holidayfl` int(11) NOT NULL COMMENT "", - `d_weekdayfl` int(11) NOT NULL COMMENT "" -) -UNIQUE KEY (`d_datekey`) -DISTRIBUTED BY HASH(`d_datekey`) BUCKETS 1 -PROPERTIES ( -"replication_num" = "1" -); \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/date_delete.sql b/regression-test/suites/mtmv_p0/ssb/ddl/date_delete.sql deleted file mode 100644 index 3841d4617a..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/date_delete.sql +++ /dev/null @@ -1 +0,0 @@ -truncate table `dates`; \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/lineorder_create.sql b/regression-test/suites/mtmv_p0/ssb/ddl/lineorder_create.sql deleted file mode 100644 index c066ea8a0d..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/lineorder_create.sql +++ /dev/null @@ -1,24 +0,0 @@ -CREATE TABLE IF NOT EXISTS `lineorder` ( - `lo_orderkey` bigint(20) NOT NULL COMMENT "", - `lo_linenumber` bigint(20) NOT NULL COMMENT "", - `lo_custkey` int(11) NOT NULL COMMENT "", - `lo_partkey` int(11) NOT NULL COMMENT "", - `lo_suppkey` int(11) NOT NULL COMMENT "", - `lo_orderdate` int(11) NOT NULL COMMENT "", - `lo_orderpriority` varchar(16) NOT NULL COMMENT "", - `lo_shippriority` int(11) NOT NULL COMMENT "", - `lo_quantity` bigint(20) NOT NULL COMMENT "", - `lo_extendedprice` bigint(20) NOT NULL COMMENT "", - `lo_ordtotalprice` bigint(20) NOT NULL COMMENT "", - `lo_discount` bigint(20) NOT NULL COMMENT "", - `lo_revenue` bigint(20) NOT NULL COMMENT "", - `lo_supplycost` bigint(20) NOT NULL COMMENT "", - `lo_tax` bigint(20) NOT NULL COMMENT "", - `lo_commitdate` bigint(20) NOT NULL COMMENT "", - `lo_shipmode` varchar(11) NOT NULL COMMENT "" -) -UNIQUE KEY (`lo_orderkey`, `lo_linenumber`) -DISTRIBUTED BY HASH(`lo_orderkey`) BUCKETS 1 -PROPERTIES ( -"replication_num" = "1" -); \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/lineorder_delete.sql b/regression-test/suites/mtmv_p0/ssb/ddl/lineorder_delete.sql deleted file mode 100644 index 329e040060..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/lineorder_delete.sql +++ /dev/null @@ -1 +0,0 @@ -truncate table lineorder; \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/lineorder_flat_create.sql b/regression-test/suites/mtmv_p0/ssb/ddl/lineorder_flat_create.sql deleted file mode 100644 index 35faee0786..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/lineorder_flat_create.sql +++ /dev/null @@ -1,45 +0,0 @@ -CREATE TABLE IF NOT EXISTS `lineorder_flat` ( - `LO_ORDERDATE` date NOT NULL COMMENT "", - `LO_ORDERKEY` int(11) NOT NULL COMMENT "", - `LO_LINENUMBER` tinyint(4) NOT NULL COMMENT "", - `LO_CUSTKEY` int(11) NOT NULL COMMENT "", - `LO_PARTKEY` int(11) NOT NULL COMMENT "", - `LO_SUPPKEY` int(11) NOT NULL COMMENT "", - `LO_ORDERPRIORITY` varchar(100) NOT NULL COMMENT "", - `LO_SHIPPRIORITY` tinyint(4) NOT NULL COMMENT "", - `LO_QUANTITY` tinyint(4) NOT NULL COMMENT "", - `LO_EXTENDEDPRICE` int(11) NOT NULL COMMENT "", - `LO_ORDTOTALPRICE` int(11) NOT NULL COMMENT "", - `LO_DISCOUNT` tinyint(4) NOT NULL COMMENT "", - `LO_REVENUE` int(11) NOT NULL COMMENT "", - `LO_SUPPLYCOST` int(11) NOT NULL COMMENT "", - `LO_TAX` tinyint(4) NOT NULL COMMENT "", - `LO_COMMITDATE` date NOT NULL COMMENT "", - `LO_SHIPMODE` varchar(100) NOT NULL COMMENT "", - `C_NAME` varchar(100) NOT NULL COMMENT "", - `C_ADDRESS` varchar(100) NOT NULL COMMENT "", - `C_CITY` varchar(100) NOT NULL COMMENT "", - `C_NATION` varchar(100) NOT NULL COMMENT "", - `C_REGION` varchar(100) NOT NULL COMMENT "", - `C_PHONE` varchar(100) NOT NULL COMMENT "", - `C_MKTSEGMENT` varchar(100) NOT NULL COMMENT "", - `S_NAME` varchar(100) NOT NULL COMMENT "", - `S_ADDRESS` varchar(100) NOT NULL COMMENT "", - `S_CITY` varchar(100) NOT NULL COMMENT "", - `S_NATION` varchar(100) NOT NULL COMMENT "", - `S_REGION` varchar(100) NOT NULL COMMENT "", - `S_PHONE` varchar(100) NOT NULL COMMENT "", - `P_NAME` varchar(100) NOT NULL COMMENT "", - `P_MFGR` varchar(100) NOT NULL COMMENT "", - `P_CATEGORY` varchar(100) NOT NULL COMMENT "", - `P_BRAND` varchar(100) NOT NULL COMMENT "", - `P_COLOR` varchar(100) NOT NULL COMMENT "", - `P_TYPE` varchar(100) NOT NULL COMMENT "", - `P_SIZE` tinyint(4) NOT NULL COMMENT "", - `P_CONTAINER` varchar(100) NOT NULL COMMENT "" -) ENGINE=OLAP -UNIQUE KEY(`LO_ORDERDATE`, `LO_ORDERKEY`, `LO_LINENUMBER`) -DISTRIBUTED BY HASH(`LO_ORDERKEY`) BUCKETS 1 -PROPERTIES ( -"replication_num" = "1" -); \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/lineorder_flat_delete.sql b/regression-test/suites/mtmv_p0/ssb/ddl/lineorder_flat_delete.sql deleted file mode 100644 index 13f6c32bb6..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/lineorder_flat_delete.sql +++ /dev/null @@ -1 +0,0 @@ -truncate table lineorder_flat; \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/part_create.sql b/regression-test/suites/mtmv_p0/ssb/ddl/part_create.sql deleted file mode 100644 index 2405026a3c..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/part_create.sql +++ /dev/null @@ -1,16 +0,0 @@ -CREATE TABLE IF NOT EXISTS `part` ( - `p_partkey` int(11) NOT NULL COMMENT "", - `p_name` varchar(23) NOT NULL COMMENT "", - `p_mfgr` varchar(7) NOT NULL COMMENT "", - `p_category` varchar(8) NOT NULL COMMENT "", - `p_brand` varchar(10) NOT NULL COMMENT "", - `p_color` varchar(12) NOT NULL COMMENT "", - `p_type` varchar(26) NOT NULL COMMENT "", - `p_size` int(11) NOT NULL COMMENT "", - `p_container` varchar(11) NOT NULL COMMENT "" -) -UNIQUE KEY (`p_partkey`) -DISTRIBUTED BY HASH(`p_partkey`) BUCKETS 1 -PROPERTIES ( -"replication_num" = "1" -); \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/part_delete.sql b/regression-test/suites/mtmv_p0/ssb/ddl/part_delete.sql deleted file mode 100644 index 02c6abd253..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/part_delete.sql +++ /dev/null @@ -1 +0,0 @@ -truncate table `part`; \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_flat_create.sql b/regression-test/suites/mtmv_p0/ssb/ddl/ssb_flat_create.sql deleted file mode 100644 index e5cbc1a556..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_flat_create.sql +++ /dev/null @@ -1,52 +0,0 @@ -CREATE MATERIALIZED VIEW ssb_flat -BUILD IMMEDIATE REFRESH COMPLETE -DISTRIBUTED BY HASH(lo_orderkey) BUCKETS 6 -PROPERTIES ('replication_num' = '1') -AS -SELECT - LO_ORDERDATE, - LO_ORDERKEY, - LO_LINENUMBER, - LO_CUSTKEY, - LO_PARTKEY, - LO_SUPPKEY, - LO_ORDERPRIORITY, - LO_SHIPPRIORITY, - LO_QUANTITY, - LO_EXTENDEDPRICE, - LO_ORDTOTALPRICE, - LO_DISCOUNT, - LO_REVENUE, - LO_SUPPLYCOST, - LO_TAX, - LO_COMMITDATE, - LO_SHIPMODE, - C_NAME, - C_ADDRESS, - C_CITY, - C_NATION, - C_REGION, - C_PHONE, - C_MKTSEGMENT, - S_NAME, - S_ADDRESS, - S_CITY, - S_NATION, - S_REGION, - S_PHONE, - P_NAME, - P_MFGR, - P_CATEGORY, - P_BRAND, - P_COLOR, - P_TYPE, - P_SIZE, - P_CONTAINER -FROM -lineorder as l -INNER JOIN customer c -ON (c.c_custkey = l.lo_custkey) -INNER JOIN supplier s -ON (s.s_suppkey = l.lo_suppkey) -INNER JOIN part p -ON (p.p_partkey = l.lo_partkey); \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q11_create.sql b/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q11_create.sql deleted file mode 100644 index fdf1a1462f..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q11_create.sql +++ /dev/null @@ -1,12 +0,0 @@ -CREATE MATERIALIZED VIEW ssb_q11 -BUILD IMMEDIATE REFRESH COMPLETE -DISTRIBUTED BY HASH(REVENUE) BUCKETS 6 -PROPERTIES ('replication_num' = '1') -AS -SELECT SUM(lo_extendedprice * lo_discount) AS REVENUE -FROM lineorder, dates -WHERE - lo_orderdate = d_datekey - AND d_year = 1993 - AND lo_discount BETWEEN 1 AND 3 - AND lo_quantity < 25; \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q12_create.sql b/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q12_create.sql deleted file mode 100644 index fcf1fbd931..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q12_create.sql +++ /dev/null @@ -1,12 +0,0 @@ -CREATE MATERIALIZED VIEW ssb_q12 -BUILD IMMEDIATE REFRESH COMPLETE -DISTRIBUTED BY HASH(REVENUE) BUCKETS 6 -PROPERTIES ('replication_num' = '1') -AS -SELECT SUM(lo_extendedprice * lo_discount) AS REVENUE -FROM lineorder, dates -WHERE - lo_orderdate = d_datekey - AND d_year = 1993 - AND lo_discount BETWEEN 1 AND 3 - AND lo_quantity < 25; \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q13_create.sql b/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q13_create.sql deleted file mode 100644 index 2da36bc129..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q13_create.sql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE MATERIALIZED VIEW ssb_q13 -BUILD IMMEDIATE REFRESH COMPLETE -DISTRIBUTED BY HASH(REVENUE) BUCKETS 6 -PROPERTIES ('replication_num' = '1') -AS -SELECT - SUM(lo_extendedprice * lo_discount) AS REVENUE -FROM lineorder, dates -WHERE - lo_orderdate = d_datekey - AND d_weeknuminyear = 6 - AND d_year = 1994 - AND lo_discount BETWEEN 5 AND 7 - AND lo_quantity BETWEEN 26 AND 35; \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q21_create.sql b/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q21_create.sql deleted file mode 100644 index 9f0ffc7e10..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q21_create.sql +++ /dev/null @@ -1,15 +0,0 @@ -CREATE MATERIALIZED VIEW ssb_q21 -BUILD IMMEDIATE REFRESH COMPLETE -DISTRIBUTED BY HASH(p_brand) BUCKETS 16 -PROPERTIES ('replication_num' = '1') -AS -SELECT SUM(lo_revenue) as revenue, d_year, p_brand -FROM lineorder, dates, part, supplier -WHERE - lo_orderdate = d_datekey - AND lo_partkey = p_partkey - AND lo_suppkey = s_suppkey - AND p_category = 'MFGR#12' - AND s_region = 'AMERICA' -GROUP BY d_year, p_brand -ORDER BY p_brand; \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q22_create.sql b/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q22_create.sql deleted file mode 100644 index 481586e1f4..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q22_create.sql +++ /dev/null @@ -1,15 +0,0 @@ -CREATE MATERIALIZED VIEW ssb_q22 -BUILD IMMEDIATE REFRESH COMPLETE -DISTRIBUTED BY HASH(p_brand) BUCKETS 6 -PROPERTIES ('replication_num' = '1') -AS -SELECT SUM(lo_revenue) as lo_revenue, d_year, p_brand -FROM lineorder, dates, part, supplier -WHERE - lo_orderdate = d_datekey - AND lo_partkey = p_partkey - AND lo_suppkey = s_suppkey - AND p_brand BETWEEN 'MFGR#2221' AND 'MFGR#2228' - AND s_region = 'ASIA' -GROUP BY d_year, p_brand -ORDER BY d_year, p_brand; \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q23_create.sql b/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q23_create.sql deleted file mode 100644 index dbaf3bb234..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q23_create.sql +++ /dev/null @@ -1,15 +0,0 @@ -CREATE MATERIALIZED VIEW ssb_q23 -BUILD IMMEDIATE REFRESH COMPLETE -DISTRIBUTED BY HASH(p_brand) BUCKETS 6 -PROPERTIES ('replication_num' = '1') -AS -SELECT SUM(lo_revenue) as revenue, d_year, p_brand -FROM lineorder, dates, part, supplier -WHERE - lo_orderdate = d_datekey - AND lo_partkey = p_partkey - AND lo_suppkey = s_suppkey - AND p_brand = 'MFGR#2239' - AND s_region = 'EUROPE' -GROUP BY d_year, p_brand -ORDER BY d_year, p_brand; \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q31_create.sql b/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q31_create.sql deleted file mode 100644 index e757cca418..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q31_create.sql +++ /dev/null @@ -1,21 +0,0 @@ -CREATE MATERIALIZED VIEW ssb_q31 -BUILD IMMEDIATE REFRESH COMPLETE -DISTRIBUTED BY HASH(c_nation, s_nation) BUCKETS 6 -PROPERTIES ('replication_num' = '1') -AS -SELECT - c_nation, - s_nation, - d_year, - SUM(lo_revenue) AS REVENUE -FROM customer, lineorder, supplier, dates -WHERE - lo_custkey = c_custkey - AND lo_suppkey = s_suppkey - AND lo_orderdate = d_datekey - AND c_region = 'ASIA' - AND s_region = 'ASIA' - AND d_year >= 1992 - AND d_year <= 1997 -GROUP BY c_nation, s_nation, d_year -ORDER BY d_year ASC, REVENUE DESC; \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q32_create.sql b/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q32_create.sql deleted file mode 100644 index fdbcd11d05..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q32_create.sql +++ /dev/null @@ -1,21 +0,0 @@ -CREATE MATERIALIZED VIEW ssb_q32 -BUILD IMMEDIATE REFRESH COMPLETE -DISTRIBUTED BY HASH(c_city, s_city) BUCKETS 6 -PROPERTIES ('replication_num' = '1') -AS -SELECT - c_city, - s_city, - d_year, - SUM(lo_revenue) AS REVENUE -FROM customer, lineorder, supplier, dates -WHERE - lo_custkey = c_custkey - AND lo_suppkey = s_suppkey - AND lo_orderdate = d_datekey - AND c_nation = 'UNITED STATES' - AND s_nation = 'UNITED STATES' - AND d_year >= 1992 - AND d_year <= 1997 -GROUP BY c_city, s_city, d_year -ORDER BY d_year ASC, REVENUE DESC; \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q33_create.sql b/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q33_create.sql deleted file mode 100644 index a23ef06708..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q33_create.sql +++ /dev/null @@ -1,27 +0,0 @@ -CREATE MATERIALIZED VIEW ssb_q33 -BUILD IMMEDIATE REFRESH COMPLETE -DISTRIBUTED BY HASH(c_city, s_city) BUCKETS 6 -PROPERTIES ('replication_num' = '1') -AS -SELECT - c_city, - s_city, - d_year, - SUM(lo_revenue) AS REVENUE -FROM customer, lineorder, supplier, dates -WHERE - lo_custkey = c_custkey - AND lo_suppkey = s_suppkey - AND lo_orderdate = d_datekey - AND ( - c_city = 'UNITED KI1' - OR c_city = 'UNITED KI5' - ) - AND ( - s_city = 'UNITED KI1' - OR s_city = 'UNITED KI5' - ) - AND d_year >= 1992 - AND d_year <= 1997 -GROUP BY c_city, s_city, d_year -ORDER BY d_year ASC, REVENUE DESC; \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q34_create.sql b/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q34_create.sql deleted file mode 100644 index 452fa67a79..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q34_create.sql +++ /dev/null @@ -1,26 +0,0 @@ -CREATE MATERIALIZED VIEW ssb_q34 -BUILD IMMEDIATE REFRESH COMPLETE -DISTRIBUTED BY HASH(c_city, s_city) BUCKETS 6 -PROPERTIES ('replication_num' = '1') -AS -SELECT - c_city, - s_city, - d_year, - SUM(lo_revenue) AS REVENUE -FROM customer, lineorder, supplier, dates -WHERE - lo_custkey = c_custkey - AND lo_suppkey = s_suppkey - AND lo_orderdate = d_datekey - AND ( - c_city = 'UNITED KI1' - OR c_city = 'UNITED KI5' - ) - AND ( - s_city = 'UNITED KI1' - OR s_city = 'UNITED KI5' - ) - AND d_yearmonth = 'Dec1997' -GROUP BY c_city, s_city, d_year -ORDER BY d_year ASC, REVENUE DESC; \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q41_create.sql b/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q41_create.sql deleted file mode 100644 index 33626acf62..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q41_create.sql +++ /dev/null @@ -1,23 +0,0 @@ -CREATE MATERIALIZED VIEW ssb_q41 -BUILD IMMEDIATE REFRESH COMPLETE -DISTRIBUTED BY HASH(c_nation) BUCKETS 6 -PROPERTIES ('replication_num' = '1') -AS -SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=4, parallel_pipeline_task_num=4, batch_size=4096) */ - d_year, - c_nation, - SUM(lo_revenue - lo_supplycost) AS PROFIT -FROM dates, customer, supplier, part, lineorder -WHERE - lo_custkey = c_custkey - AND lo_suppkey = s_suppkey - AND lo_partkey = p_partkey - AND lo_orderdate = d_datekey - AND c_region = 'AMERICA' - AND s_region = 'AMERICA' - AND ( - p_mfgr = 'MFGR#1' - OR p_mfgr = 'MFGR#2' - ) -GROUP BY d_year, c_nation -ORDER BY d_year, c_nation; \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q42_create.sql b/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q42_create.sql deleted file mode 100644 index a6db33e951..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q42_create.sql +++ /dev/null @@ -1,28 +0,0 @@ -CREATE MATERIALIZED VIEW ssb_q42 -BUILD IMMEDIATE REFRESH COMPLETE -DISTRIBUTED BY HASH(s_nation, p_category) BUCKETS 6 -PROPERTIES ('replication_num' = '1') -AS -SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=2, parallel_pipeline_task_num=2, batch_size=4096) */ - d_year, - s_nation, - p_category, - SUM(lo_revenue - lo_supplycost) AS PROFIT -FROM dates, customer, supplier, part, lineorder -WHERE - lo_custkey = c_custkey - AND lo_suppkey = s_suppkey - AND lo_partkey = p_partkey - AND lo_orderdate = d_datekey - AND c_region = 'AMERICA' - AND s_region = 'AMERICA' - AND ( - d_year = 1997 - OR d_year = 1998 - ) - AND ( - p_mfgr = 'MFGR#1' - OR p_mfgr = 'MFGR#2' - ) -GROUP BY d_year, s_nation, p_category -ORDER BY d_year, s_nation, p_category; \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q43_create.sql b/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q43_create.sql deleted file mode 100644 index 7f8d4311cb..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/ssb_q43_create.sql +++ /dev/null @@ -1,24 +0,0 @@ -CREATE MATERIALIZED VIEW ssb_q43 -BUILD IMMEDIATE REFRESH COMPLETE -DISTRIBUTED BY HASH(s_city, p_brand) BUCKETS 6 -PROPERTIES ('replication_num' = '1') -AS -SELECT /*+SET_VAR(parallel_fragment_exec_instance_num=2, parallel_pipeline_task_num=2, batch_size=4096) */ - d_year, - s_city, - p_brand, - SUM(lo_revenue - lo_supplycost) AS PROFIT -FROM dates, customer, supplier, part, lineorder -WHERE - lo_custkey = c_custkey - AND lo_suppkey = s_suppkey - AND lo_partkey = p_partkey - AND lo_orderdate = d_datekey - AND s_nation = 'UNITED STATES' - AND ( - d_year = 1997 - OR d_year = 1998 - ) - AND p_category = 'MFGR#14' -GROUP BY d_year, s_city, p_brand -ORDER BY d_year, s_city, p_brand; diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/supplier_create.sql b/regression-test/suites/mtmv_p0/ssb/ddl/supplier_create.sql deleted file mode 100644 index 1580a8fcca..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/supplier_create.sql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE TABLE IF NOT EXISTS `supplier` ( - `s_suppkey` int(11) NOT NULL COMMENT "", - `s_name` varchar(26) NOT NULL COMMENT "", - `s_address` varchar(26) NOT NULL COMMENT "", - `s_city` varchar(11) NOT NULL COMMENT "", - `s_nation` varchar(16) NOT NULL COMMENT "", - `s_region` varchar(13) NOT NULL COMMENT "", - `s_phone` varchar(16) NOT NULL COMMENT "" -) -UNIQUE KEY (`s_suppkey`) -DISTRIBUTED BY HASH(`s_suppkey`) BUCKETS 1 -PROPERTIES ( -"replication_num" = "1" -); \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/ssb/ddl/supplier_delete.sql b/regression-test/suites/mtmv_p0/ssb/ddl/supplier_delete.sql deleted file mode 100644 index 39e663134c..0000000000 --- a/regression-test/suites/mtmv_p0/ssb/ddl/supplier_delete.sql +++ /dev/null @@ -1 +0,0 @@ -truncate table `supplier`; \ No newline at end of file diff --git a/regression-test/suites/mtmv_p0/test_alter_mtmv.groovy b/regression-test/suites/mtmv_p0/test_alter_mtmv.groovy deleted file mode 100644 index 53f90aeffa..0000000000 --- a/regression-test/suites/mtmv_p0/test_alter_mtmv.groovy +++ /dev/null @@ -1,86 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_alter_mtmv") { - def tableName = "t_test_alter_mtmv_user" - def tableNamePv = "t_test_alter_mtmv_pv" - def mvName = "multi_mv_test_alter_mtmv" - - sql """drop table if exists `${tableName}`""" - sql """drop table if exists `${tableNamePv}`""" - - sql """ - CREATE TABLE IF NOT EXISTS `${tableName}` ( - event_day DATE, - id BIGINT, - username VARCHAR(20) - ) - DISTRIBUTED BY HASH(id) BUCKETS 10 - PROPERTIES ( - "replication_num" = "1" - ); - """ - sql """ - INSERT INTO ${tableName} VALUES("2022-10-26",1,"clz"),("2022-10-28",2,"zhangsang"),("2022-10-29",3,"lisi"); - """ - sql """ - create table IF NOT EXISTS ${tableNamePv}( - event_day DATE, - id BIGINT, - pv BIGINT - ) - DISTRIBUTED BY HASH(id) BUCKETS 10 - PROPERTIES ( - "replication_num" = "1" - ); - """ - - sql """ - INSERT INTO ${tableNamePv} VALUES("2022-10-26",1,200),("2022-10-28",2,200),("2022-10-28",3,300); - """ - - sql """drop materialized view if exists ${mvName}""" - - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD IMMEDIATE REFRESH COMPLETE - KEY(username) - DISTRIBUTED BY HASH (username) buckets 1 - PROPERTIES ('replication_num' = '1') - AS - SELECT ${tableName}.username, ${tableNamePv}.pv FROM ${tableName}, ${tableNamePv} WHERE ${tableName}.id=${tableNamePv}.id; - """ - - // waiting the task to be finished. - waitingMTMVTaskFinished(mvName) - - // test alter mtmv - sql """ - alter MATERIALIZED VIEW ${mvName} REFRESH COMPLETE start with "2022-11-03 00:00:00" next 2 DAY - """ - show_job_meta = sql_meta "SHOW MTMV JOB ON ${mvName}" - def scheduleIndex = show_job_meta.indexOf(['Schedule', 'CHAR']) - - show_job_result = sql "SHOW MTMV JOB ON ${mvName}" - assertEquals 1, show_job_result.size(), show_job_result.toString() - - assertEquals 'START 2022-11-03T00:00 EVERY(2 DAYS)', show_job_result.last().get(scheduleIndex).toString(), show_job_result.last().toString() - - sql """ - DROP MATERIALIZED VIEW ${mvName} - """ -} diff --git a/regression-test/suites/mtmv_p0/test_create_both_mtmv.groovy b/regression-test/suites/mtmv_p0/test_create_both_mtmv.groovy deleted file mode 100644 index 4eb9ad2cde..0000000000 --- a/regression-test/suites/mtmv_p0/test_create_both_mtmv.groovy +++ /dev/null @@ -1,78 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_create_both_mtmv") { - def tableName = "t_test_create_both_mtmv_user" - def tableNamePv = "t_test_create_both_mtmv_user_pv" - def mvName = "multi_mv_test_create_both_mtmv" - - sql """drop table if exists `${tableName}`""" - sql """drop table if exists `${tableNamePv}`""" - - sql """ - CREATE TABLE IF NOT EXISTS `${tableName}` ( - event_day DATE, - id BIGINT, - username VARCHAR(20) - ) - DISTRIBUTED BY HASH(id) BUCKETS 10 - PROPERTIES ( - "replication_num" = "1" - ); - """ - sql """ - INSERT INTO ${tableName} VALUES("2022-10-26",1,"clz"),("2022-10-28",2,"zhangsang"),("2022-10-29",3,"lisi"); - """ - sql """ - create table IF NOT EXISTS ${tableNamePv}( - event_day DATE, - id BIGINT, - pv BIGINT - ) - DISTRIBUTED BY HASH(id) BUCKETS 10 - PROPERTIES ( - "replication_num" = "1" - ); - """ - - sql """ - INSERT INTO ${tableNamePv} VALUES("2022-10-26",1,200),("2022-10-28",2,200),("2022-10-28",3,300); - """ - - sql """drop materialized view if exists ${mvName}""" - - // test only one job created when build IMMEDIATE and start time is before now. - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD IMMEDIATE REFRESH COMPLETE - start with "2022-11-03 00:00:00" next 1 DAY - KEY(username) - DISTRIBUTED BY HASH (username) buckets 1 - PROPERTIES ('replication_num' = '1') - AS - SELECT ${tableName}.username, ${tableNamePv}.pv FROM ${tableName}, ${tableNamePv} WHERE ${tableName}.id=${tableNamePv}.id; - """ - // wait task to be finished to avoid task leak in suite. - waitingMTMVTaskFinished(mvName) - - def show_job_result = sql "SHOW MTMV JOB ON ${mvName}" - assertEquals 1, show_job_result.size(), show_job_result.toString() - - sql """ - DROP MATERIALIZED VIEW ${mvName} - """ -} diff --git a/regression-test/suites/mtmv_p0/test_create_mtmv.groovy b/regression-test/suites/mtmv_p0/test_create_mtmv.groovy deleted file mode 100644 index c3aba28a9e..0000000000 --- a/regression-test/suites/mtmv_p0/test_create_mtmv.groovy +++ /dev/null @@ -1,75 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_create_mtmv") { - def tableName = "t_test_create_mtmv_user" - def tableNamePv = "t_test_create_mtmv_user_pv" - def mvName = "multi_mv_test_create_mtmv" - - sql """drop table if exists `${tableName}`""" - sql """drop table if exists `${tableNamePv}`""" - - sql """ - CREATE TABLE IF NOT EXISTS `${tableName}` ( - event_day DATE, - id BIGINT, - username VARCHAR(20) - ) - DISTRIBUTED BY HASH(id) BUCKETS 10 - PROPERTIES ( - "replication_num" = "1" - ); - """ - sql """ - INSERT INTO ${tableName} VALUES("2022-10-26",1,"clz"),("2022-10-28",2,"zhangsang"),("2022-10-29",3,"lisi"); - """ - sql """ - create table IF NOT EXISTS ${tableNamePv}( - event_day DATE, - id BIGINT, - pv BIGINT - ) - DISTRIBUTED BY HASH(id) BUCKETS 10 - PROPERTIES ( - "replication_num" = "1" - ); - """ - - sql """ - INSERT INTO ${tableNamePv} VALUES("2022-10-26",1,200),("2022-10-28",2,200),("2022-10-28",3,300); - """ - - sql """drop materialized view if exists ${mvName}""" - - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD IMMEDIATE REFRESH COMPLETE - KEY(username) - DISTRIBUTED BY HASH (username) buckets 1 - PROPERTIES ('replication_num' = '1') - AS - SELECT ${tableName}.username, ${tableNamePv}.pv FROM ${tableName}, ${tableNamePv} WHERE ${tableName}.id=${tableNamePv}.id; - """ - - waitingMTMVTaskFinished(mvName) - - order_qt_select "SELECT * FROM ${mvName}" - - sql """ - DROP MATERIALIZED VIEW ${mvName} - """ -} diff --git a/regression-test/suites/mtmv_p0/test_refresh_mtmv.groovy b/regression-test/suites/mtmv_p0/test_refresh_mtmv.groovy deleted file mode 100644 index 70ee205d35..0000000000 --- a/regression-test/suites/mtmv_p0/test_refresh_mtmv.groovy +++ /dev/null @@ -1,110 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_refresh_mtmv") { - def tableName = "t_test_refresh_mtmv_user" - def tableNamePv = "t_test_refresh_mtmv_user_pv" - def mvName = "multi_mv_test_refresh_mtmv" - def mvNameDemand = "multi_mv_test_refresh_demand_mtmv" - - sql """drop table if exists `${tableName}`""" - sql """drop table if exists `${tableNamePv}`""" - - sql """ - CREATE TABLE IF NOT EXISTS `${tableName}` ( - event_day DATE, - id BIGINT, - username VARCHAR(20) - ) - DISTRIBUTED BY HASH(id) BUCKETS 10 - PROPERTIES ( - "replication_num" = "1" - ); - """ - sql """ - INSERT INTO ${tableName} VALUES("2022-10-26",1,"clz"),("2022-10-28",2,"zhangsang"),("2022-10-29",3,"lisi"); - """ - sql """ - create table IF NOT EXISTS ${tableNamePv}( - event_day DATE, - id BIGINT, - pv BIGINT - ) - DISTRIBUTED BY HASH(id) BUCKETS 10 - PROPERTIES ( - "replication_num" = "1" - ); - """ - - sql """ - INSERT INTO ${tableNamePv} VALUES("2022-10-26",1,200),("2022-10-28",2,200),("2022-10-28",3,300); - """ - - sql """drop materialized view if exists ${mvName}""" - - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD IMMEDIATE REFRESH COMPLETE - KEY(username) - DISTRIBUTED BY HASH (username) buckets 1 - PROPERTIES ('replication_num' = '1') - AS - SELECT ${tableName}.username, ${tableNamePv}.pv FROM ${tableName}, ${tableNamePv} WHERE ${tableName}.id=${tableNamePv}.id; - """ - - // waiting the task to be finished. - waitingMTMVTaskFinished(mvName) - - order_qt_select "SELECT * FROM ${mvName}" - - // test REFRESH make sure only define one mv and already run a task. - sql """ - REFRESH MATERIALIZED VIEW ${mvName} COMPLETE - """ - waitingMTMVTaskFinished(mvName) - - def show_task_result = sql "SHOW MTMV TASK ON ${mvName}" - assertEquals 2, show_task_result.size(), show_task_result.toString() - - sql """ - DROP MATERIALIZED VIEW ${mvName} - """ - - sql """drop materialized view if exists ${mvNameDemand}""" - - sql """ - CREATE MATERIALIZED VIEW ${mvNameDemand} - BUILD DEFERRED REFRESH COMPLETE ON DEMAND - KEY(username) - DISTRIBUTED BY HASH (username) buckets 1 - PROPERTIES ('replication_num' = '1') - AS - SELECT ${tableName}.username, ${tableNamePv}.pv FROM ${tableName}, ${tableNamePv} WHERE ${tableName}.id=${tableNamePv}.id; - """ - - sql """ - REFRESH MATERIALIZED VIEW ${mvNameDemand} COMPLETE - """ - waitingMTMVTaskFinished(mvNameDemand) - - show_task_result = sql "SHOW MTMV TASK ON ${mvNameDemand}" - assertEquals 1, show_task_result.size(), show_task_result.toString() - - sql """ - DROP MATERIALIZED VIEW ${mvNameDemand} - """ -}