From 96413e679d547ee144308d663c8f9c01b32f21db Mon Sep 17 00:00:00 2001 From: Lijia Liu Date: Wed, 31 Jul 2024 22:32:28 +0800 Subject: [PATCH] [branch-2.1](mtmv) Support read sync materialized view in async materialized view (#38462) ## Proposed changes pick #37396 --------- Co-authored-by: liutang123 --- .../exploration/mv/MaterializedViewUtils.java | 9 ++ .../rollup_p0/test_create_mv_and_mtmv.out | 14 +++ .../rollup_p0/test_create_mv_and_mtmv.groovy | 109 ++++++++++++++++++ 3 files changed, 132 insertions(+) create mode 100644 regression-test/data/rollup_p0/test_create_mv_and_mtmv.out create mode 100644 regression-test/suites/rollup_p0/test_create_mv_and_mtmv.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 937f90c61e..f1ce770de2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -17,6 +17,8 @@ package org.apache.doris.nereids.rules.exploration.mv; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.SlotRef; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.PartitionType; @@ -405,6 +407,13 @@ public class MaterializedViewUtils { return null; } Column mvReferenceColumn = contextPartitionColumn.getColumn().get(); + Expr definExpr = mvReferenceColumn.getDefineExpr(); + if (definExpr instanceof SlotRef) { + Column referenceRollupColumn = ((SlotRef) definExpr).getColumn(); + if (referenceRollupColumn != null) { + mvReferenceColumn = referenceRollupColumn; + } + } if (partitionColumnSet.contains(mvReferenceColumn) && (!mvReferenceColumn.isAllowNull() || relatedTable.isPartitionColumnAllowNull())) { context.addTableColumn(table, mvReferenceColumn); diff --git a/regression-test/data/rollup_p0/test_create_mv_and_mtmv.out b/regression-test/data/rollup_p0/test_create_mv_and_mtmv.out new file mode 100644 index 0000000000..d4ea853add --- /dev/null +++ b/regression-test/data/rollup_p0/test_create_mv_and_mtmv.out @@ -0,0 +1,14 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !mtmv_init -- +2024-07-02 a 1 +2024-07-03 b 1 + +-- !insert_into_partial_old_partition -- +2024-07-02 a 1 +2024-07-03 b 2 + +-- !insert_into_partial_new_partition -- +2024-07-02 a 1 +2024-07-03 b 2 +2024-07-04 b 1 + diff --git a/regression-test/suites/rollup_p0/test_create_mv_and_mtmv.groovy b/regression-test/suites/rollup_p0/test_create_mv_and_mtmv.groovy new file mode 100644 index 0000000000..43f08c72db --- /dev/null +++ b/regression-test/suites/rollup_p0/test_create_mv_and_mtmv.groovy @@ -0,0 +1,109 @@ +import java.text.SimpleDateFormat + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_create_mv_and_mtmt") { + def tableName = "test_create_mv_and_mtmt_advertiser_view_record" + def mvName = "test_create_mv_and_mtmt_advertiser_uv" + def mtmvName = "test_create_mv_and_mtmt_advertiser_uv_mtmv" + sql """ + CREATE TABLE ${tableName}( + time date not null, + advertiser varchar(10), + dt date not null, + channel varchar(10), + user_id int) + DUPLICATE KEY(`time`, `advertiser`) + PARTITION BY RANGE (dt)(FROM ("2024-07-02") TO ("2024-07-05") INTERVAL 1 DAY) + -- AUTO PARTITION BY RANGE (date_trunc(`time`, 'day'))() + distributed BY hash(time) + properties("replication_allocation" = "tag.location.default: 1"); + """ + sql """ insert into ${tableName} values("2024-07-02",'a', "2024-07-02", 'a',1); """ + sql """ insert into ${tableName} values("2024-07-03",'b', "2024-07-03", 'b',1); """ + + createMV(""" + CREATE materialized VIEW ${mvName} AS + SELECT advertiser, + channel, + dt, + bitmap_union(to_bitmap(user_id)) + FROM ${tableName} + GROUP BY advertiser, + channel, + dt; + + """) + explain { + sql(""" + SELECT dt,advertiser, + count(DISTINCT user_id) + FROM ${tableName} + GROUP BY dt,advertiser""") + contains "(${mvName})" + } + def refreshTime = System.currentTimeMillis() / 1000L + sql """ + CREATE MATERIALIZED VIEW ${mtmvName} + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + partition by(dt) + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ('replication_num' = '1') + AS + select dt, advertiser, count(distinct user_id) + from ${tableName} + group by dt, advertiser; + """ + def wait_mtmv_refresh_finish = { refreshMode-> + for (int loop = 0; loop < 300; loop++) { + Thread.sleep(1200) + boolean finished = true; + def result = sql """ +select * from tasks('type'='mv') +where MvName='${mtmvName}' +AND Status = 'SUCCESS' +AND CreateTime >= from_unixtime(${refreshTime}) +AND RefreshMode = '${refreshMode}';""" + finished = (result.size() == 1) + if (finished) { + def sqlResult = result[0] + log.info("refresh result: ${sqlResult}") + return; + } + } + throw new Exception("Wait mtmv ${mtmvName} finish timeout.") + } + wait_mtmv_refresh_finish("COMPLETE") + qt_mtmv_init """ SELECT * FROM ${mtmvName} ORDER BY dt, advertiser""" + + sql """INSERT INTO ${tableName} VALUES("2024-07-03",'b', "2024-07-03", 'b',2);""" + refreshTime = System.currentTimeMillis() / 1000L + sql """REFRESH MATERIALIZED VIEW ${mtmvName} AUTO;""" + wait_mtmv_refresh_finish("PARTIAL") + qt_insert_into_partial_old_partition """ SELECT * FROM ${mtmvName} ORDER BY dt, advertiser""" + + sql """INSERT INTO ${tableName} VALUES("2024-07-04",'b', "2024-07-04", 'a',1);""" + refreshTime = System.currentTimeMillis() / 1000L + sql """REFRESH MATERIALIZED VIEW ${mtmvName} AUTO;""" + wait_mtmv_refresh_finish("PARTIAL") + qt_insert_into_partial_new_partition """ SELECT * FROM ${mtmvName} ORDER BY dt, advertiser""" + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ DROP MATERIALIZED VIEW ${mtmvName} """ + +}