From 9b4354fcb7b0d0aeabb5a447fcba9c56fc68e3d1 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Sat, 10 Aug 2024 18:26:56 +0800 Subject: [PATCH] =?UTF-8?q?[fix](mtmv)=20second=20level=20MTMV=20always=20?= =?UTF-8?q?refresh=20all=20partition=20by=20mistake=E2=80=A6=20(#39167)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pick: https://github.com/apache/doris/pull/38698 --- .../doris/job/extensions/mtmv/MTMVTask.java | 10 +++--- .../apache/doris/mtmv/MTMVPartitionUtil.java | 5 +-- .../apache/doris/mtmv/MTMVRewriteUtil.java | 2 +- .../java/org/apache/doris/mtmv/MTMVUtil.java | 2 +- .../doris/mtmv/MTMVPartitionUtilTest.java | 2 +- .../data/mtmv_p0/test_multi_level_mtmv.out | 6 ++++ .../mtmv_p0/test_multi_level_mtmv.groovy | 31 +++++++++++++++---- 7 files changed, 43 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index b50d755951..0207301c2d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -196,7 +196,7 @@ public class MTMVTask extends AbstractTask { .subList(start, end > needRefreshPartitions.size() ? needRefreshPartitions.size() : end)); // need get names before exec Map execPartitionSnapshots = MTMVPartitionUtil - .generatePartitionSnapshots(mtmv, relation.getBaseTables(), execPartitionNames, + .generatePartitionSnapshots(mtmv, relation.getBaseTablesOneLevel(), execPartitionNames, partitionMappings); exec(ctx, execPartitionNames, tableWithPartKey); completedPartitions.addAll(execPartitionNames); @@ -288,7 +288,7 @@ public class MTMVTask extends AbstractTask { * @throws DdlException */ private void refreshHmsTable() throws AnalysisException, DdlException { - for (BaseTableInfo tableInfo : relation.getBaseTables()) { + for (BaseTableInfo tableInfo : relation.getBaseTablesOneLevel()) { TableIf tableIf = MTMVUtil.getTable(tableInfo); if (tableIf instanceof HMSExternalTable) { HMSExternalTable hmsTable = (HMSExternalTable) tableIf; @@ -450,7 +450,8 @@ public class MTMVTask extends AbstractTask { // check if data is fresh // We need to use a newly generated relationship and cannot retrieve it using mtmv.getRelation() // to avoid rebuilding the baseTable and causing a change in the tableId - boolean fresh = MTMVPartitionUtil.isMTMVSync(mtmv, relation.getBaseTables(), mtmv.getExcludedTriggerTables(), + boolean fresh = MTMVPartitionUtil.isMTMVSync(mtmv, relation.getBaseTablesOneLevel(), + mtmv.getExcludedTriggerTables(), partitionMappings); if (fresh) { return Lists.newArrayList(); @@ -461,7 +462,8 @@ public class MTMVTask extends AbstractTask { } // We need to use a newly generated relationship and cannot retrieve it using mtmv.getRelation() // to avoid rebuilding the baseTable and causing a change in the tableId - return MTMVPartitionUtil.getMTMVNeedRefreshPartitions(mtmv, relation.getBaseTables(), partitionMappings); + return MTMVPartitionUtil.getMTMVNeedRefreshPartitions(mtmv, relation.getBaseTablesOneLevel(), + partitionMappings); } public MTMVTaskContext getTaskContext() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java index 6595afb70f..9bf0d82146 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java @@ -201,7 +201,8 @@ public class MTMVPartitionUtil { return false; } try { - return isMTMVSync(mtmv, mtmvRelation.getBaseTables(), Sets.newHashSet(), mtmv.calculatePartitionMappings()); + return isMTMVSync(mtmv, mtmvRelation.getBaseTablesOneLevel(), Sets.newHashSet(), + mtmv.calculatePartitionMappings()); } catch (AnalysisException e) { LOG.warn("isMTMVSync failed: ", e); return false; @@ -254,7 +255,7 @@ public class MTMVPartitionUtil { Set relatedPartitionNames) throws AnalysisException { List res = Lists.newArrayList(); - for (BaseTableInfo baseTableInfo : mtmv.getRelation().getBaseTables()) { + for (BaseTableInfo baseTableInfo : mtmv.getRelation().getBaseTablesOneLevel()) { TableIf table = MTMVUtil.getTable(baseTableInfo); if (!(table instanceof MTMVRelatedTableIf)) { continue; diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java index 3516e75427..5392313ba6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java @@ -71,7 +71,7 @@ public class MTMVRewriteUtil { partitionMappings = mtmv.calculatePartitionMappings(); } if (MTMVPartitionUtil.isMTMVPartitionSync(mtmv, partition.getName(), - partitionMappings.get(partition.getName()), mtmvRelation.getBaseTables(), + partitionMappings.get(partition.getName()), mtmvRelation.getBaseTablesOneLevel(), Sets.newHashSet())) { res.add(partition); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java index 1fd56d7b6e..4868ef94a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java @@ -85,7 +85,7 @@ public class MTMVUtil { * @return */ public static boolean mtmvContainsExternalTable(MTMV mtmv) { - Set baseTables = mtmv.getRelation().getBaseTables(); + Set baseTables = mtmv.getRelation().getBaseTablesOneLevel(); for (BaseTableInfo baseTableInfo : baseTables) { if (baseTableInfo.getCtlId() != InternalCatalog.INTERNAL_CATALOG_ID) { return true; diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java index c125e548d0..d40e6db5d4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java @@ -111,7 +111,7 @@ public class MTMVPartitionUtilTest { minTimes = 0; result = true; - relation.getBaseTables(); + relation.getBaseTablesOneLevel(); minTimes = 0; result = baseTables; diff --git a/regression-test/data/mtmv_p0/test_multi_level_mtmv.out b/regression-test/data/mtmv_p0/test_multi_level_mtmv.out index 3d8a1e1b43..7543b21ffa 100644 --- a/regression-test/data/mtmv_p0/test_multi_level_mtmv.out +++ b/regression-test/data/mtmv_p0/test_multi_level_mtmv.out @@ -5,6 +5,12 @@ -- !mv2 -- 1 1 +-- !mv1_should_one_partition -- +["p_2"] + +-- !mv2_should_one_partition -- +["p_2"] + -- !status1 -- multi_level_mtmv1 SCHEMA_CHANGE SUCCESS diff --git a/regression-test/suites/mtmv_p0/test_multi_level_mtmv.groovy b/regression-test/suites/mtmv_p0/test_multi_level_mtmv.groovy index 81d4b78a3d..55689b7414 100644 --- a/regression-test/suites/mtmv_p0/test_multi_level_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_multi_level_mtmv.groovy @@ -29,6 +29,11 @@ suite("test_multi_level_mtmv") { k1 int, k2 int ) + PARTITION BY LIST(`k1`) + ( + PARTITION `p1` VALUES IN ('1'), + PARTITION `p2` VALUES IN ('2') + ) DISTRIBUTED BY HASH(k1) BUCKETS 10 PROPERTIES ( "replication_num" = "1" @@ -40,34 +45,48 @@ suite("test_multi_level_mtmv") { sql """ CREATE MATERIALIZED VIEW ${mv1} - BUILD DEFERRED REFRESH COMPLETE ON MANUAL + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(k1) DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') AS SELECT * FROM ${tableName}; """ - def jobName1 = getJobName("regression_test_mtmv_p0", mv1); sql """ REFRESH MATERIALIZED VIEW ${mv1} AUTO """ - waitingMTMVTaskFinished(jobName1) + waitingMTMVTaskFinishedByMvName(mv1) order_qt_mv1 "select * from ${mv1}" sql """ CREATE MATERIALIZED VIEW ${mv2} - BUILD DEFERRED REFRESH COMPLETE ON MANUAL + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(k1) DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') AS SELECT * FROM ${mv1}; """ - def jobName2 = getJobName("regression_test_mtmv_p0", mv2); sql """ REFRESH MATERIALIZED VIEW ${mv2} AUTO """ - waitingMTMVTaskFinished(jobName2) + waitingMTMVTaskFinishedByMvName(mv2) order_qt_mv2 "select * from ${mv2}" + sql """ + INSERT INTO ${tableName} VALUES(2,2); + """ + sql """ + REFRESH MATERIALIZED VIEW ${mv1} AUTO + """ + waitingMTMVTaskFinishedByMvName(mv1) + order_qt_mv1_should_one_partition "select NeedRefreshPartitions from tasks('type'='mv') where MvName = '${mv1}' order by CreateTime desc limit 1" + sql """ + REFRESH MATERIALIZED VIEW ${mv2} AUTO + """ + waitingMTMVTaskFinishedByMvName(mv2) + order_qt_mv2_should_one_partition "select NeedRefreshPartitions from tasks('type'='mv') where MvName = '${mv2}' order by CreateTime desc limit 1" + // drop table sql """ drop table ${tableName}