database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](mtmv) Fix exception when create materialized view with cte (#33988)

Browse Source 
Fix exception when create materialized view with cte, after this fix, can create materialized view with following
```
        CREATE MATERIALIZED VIEW mv_with_cte
            BUILD IMMEDIATE REFRESH AUTO ON MANUAL
            DISTRIBUTED BY RANDOM BUCKETS 2
            PROPERTIES ('replication_num' = '1')
            AS
            with `test_with` AS (
            select l_partkey, l_suppkey
            from lineitem
            union
            select
              ps_partkey, ps_suppkey
            from
            partsupp)
            select * from test_with;
```

this is brought from https://github.com/apache/doris/pull/28144
This commit is contained in:
seawinde
2024-04-24 22:11:30 +08:00
committed by yiguolei
parent f4deb42a80
commit a15a8e119f
3 changed files with 88 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java
View File

@ -41,10 +41,12 @@ import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.Sink;
import org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTableCommand;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.RelationUtil;
@ -197,11 +199,21 @@ public class UpdateMvByPartitionCommand extends InsertOverwriteTableCommand {
}
/**
* Add predicates on base table when mv can partition update
* Add predicates on base table when mv can partition update, Also support plan that contain cte and view
*/
public static class PredicateAdder extends DefaultPlanRewriter<Map<TableIf, Set<Expression>>> {
// record view and cte name parts, these should be ignored and visit it's actual plan
public Set<List<String>> virtualRelationNamePartSet = new HashSet<>();
@Override
public Plan visitUnboundRelation(UnboundRelation unboundRelation, Map<TableIf, Set<Expression>> predicates) {
if (predicates.isEmpty()) {
return unboundRelation;
}
if (virtualRelationNamePartSet.contains(unboundRelation.getNameParts())) {
return unboundRelation;
}
List<String> tableQualifier = RelationUtil.getQualifierName(ConnectContext.get(),
unboundRelation.getNameParts());
TableIf table = RelationUtil.getTable(tableQualifier, Env.getCurrentEnv());
@ -212,9 +224,34 @@ public class UpdateMvByPartitionCommand extends InsertOverwriteTableCommand {
return unboundRelation;
}
@Override
public Plan visitLogicalCTE(LogicalCTE<? extends Plan> cte, Map<TableIf, Set<Expression>> predicates) {
if (predicates.isEmpty()) {
return cte;
}
for (LogicalSubQueryAlias<Plan> subQueryAlias : cte.getAliasQueries()) {
this.virtualRelationNamePartSet.add(subQueryAlias.getQualifier());
subQueryAlias.children().forEach(subQuery -> subQuery.accept(this, predicates));
}
return super.visitLogicalCTE(cte, predicates);
}
@Override
public Plan visitLogicalSubQueryAlias(LogicalSubQueryAlias<? extends Plan> subQueryAlias,
Map<TableIf, Set<Expression>> predicates) {
if (predicates.isEmpty()) {
return subQueryAlias;
}
this.virtualRelationNamePartSet.add(subQueryAlias.getQualifier());
return super.visitLogicalSubQueryAlias(subQueryAlias, predicates);
}
@Override
public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation,
Map<TableIf, Set<Expression>> predicates) {
if (predicates.isEmpty()) {
return catalogRelation;
}
TableIf table = catalogRelation.getTable();
if (predicates.containsKey(table)) {
return new LogicalFilter<>(ImmutableSet.of(ExpressionUtils.or(predicates.get(table))),

6
regression-test/data/mtmv_p0/test_build_mtmv.out
View File

@ -63,3 +63,9 @@ zhangsang 200
-- !desc_mv --
field_1 VARCHAR(16) No false \N NONE
-- !query_mv_with_cte --
2 3
2 4
3 3
4 3

44
regression-test/suites/mtmv_p0/test_build_mtmv.groovy
View File

@ -625,6 +625,30 @@ suite("test_build_mtmv") {
(5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx');
"""
sql """
drop table if exists partsupp
"""
sql """
CREATE TABLE IF NOT EXISTS partsupp (
ps_partkey INTEGER NOT NULL,
ps_suppkey INTEGER NOT NULL,
ps_availqty INTEGER NOT NULL,
ps_supplycost DECIMALV3(15,2) NOT NULL,
ps_comment VARCHAR(199) NOT NULL
)
DUPLICATE KEY(ps_partkey, ps_suppkey)
DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3
PROPERTIES (
"replication_num" = "1"
)"""
sql """
insert into partsupp values
(2, 3, 9, 10.01, 'supply1'),
(2, 3, 10, 11.01, 'supply2');
"""
sql """DROP MATERIALIZED VIEW IF EXISTS test_varchar_literal_mv;"""
sql """
CREATE MATERIALIZED VIEW test_varchar_literal_mv
@ -635,4 +659,24 @@ suite("test_build_mtmv") {
select case when l_orderkey > 1 then "一二三四" else "五六七八" end as field_1 from lineitem;
"""
qt_desc_mv """desc test_varchar_literal_mv;"""
sql """DROP MATERIALIZED VIEW IF EXISTS mv_with_cte;"""
sql """
CREATE MATERIALIZED VIEW mv_with_cte
BUILD IMMEDIATE REFRESH AUTO ON MANUAL
DISTRIBUTED BY RANDOM BUCKETS 2
PROPERTIES ('replication_num' = '1')
AS
with `test_with` AS (
select l_partkey, l_suppkey
from lineitem
union
select
ps_partkey, ps_suppkey
from
partsupp)
select * from test_with;
"""
waitingMTMVTaskFinished(getJobName("regression_test_mtmv_p0", "mv_with_cte"))
order_qt_query_mv_with_cte """select * from mv_with_cte;"""
}