diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index 5a47bb07fd..b37b04d802 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.common.Pair; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanSplitContext; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; @@ -370,7 +371,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate * slot reference equals currently. */ @Override - protected boolean checkPattern(StructInfo structInfo) { + protected boolean checkPattern(StructInfo structInfo, CascadesContext cascadesContext) { PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET); // if query or mv contains more then one top aggregate, should fail return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java index 2a05fecd33..3b20cefbba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.rules.exploration.mv; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.trees.expressions.Alias; @@ -74,7 +75,7 @@ public abstract class AbstractMaterializedViewJoinRule extends AbstractMateriali * Join condition should be slot reference equals currently. */ @Override - protected boolean checkPattern(StructInfo structInfo) { + protected boolean checkPattern(StructInfo structInfo, CascadesContext cascadesContext) { PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET); return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext) && !checkContext.isContainsTopAggregate(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 2274ad441f..8442d2485c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -66,6 +66,8 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.ArrayList; import java.util.BitSet; @@ -82,6 +84,8 @@ import java.util.stream.Collectors; * The abstract class for all materialized view rules */ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFactory { + + public static final Logger LOG = LogManager.getLogger(AbstractMaterializedViewRule.class); public static final Set SUPPORTED_JOIN_TYPE_SET = ImmutableSet.of( JoinType.INNER_JOIN, JoinType.LEFT_OUTER_JOIN, @@ -142,7 +146,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac List uncheckedStructInfos = MaterializedViewUtils.extractStructInfo(queryPlan, cascadesContext, materializedViewTableSet); uncheckedStructInfos.forEach(queryStructInfo -> { - boolean valid = checkPattern(queryStructInfo) && queryStructInfo.isValid(); + boolean valid = checkPattern(queryStructInfo, cascadesContext) && queryStructInfo.isValid(); if (!valid) { cascadesContext.getMaterializationContexts().forEach(ctx -> ctx.recordFailReason(queryStructInfo, "Query struct info is invalid", @@ -178,6 +182,13 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac "Query to view table mapping is null", () -> ""); return rewriteResults; } + int materializedViewRelationMappingMaxCount = cascadesContext.getConnectContext().getSessionVariable() + .getMaterializedViewRelationMappingMaxCount(); + if (queryToViewTableMappings.size() > materializedViewRelationMappingMaxCount) { + LOG.warn("queryToViewTableMappings is over limit and be intercepted"); + queryToViewTableMappings = queryToViewTableMappings.subList(0, materializedViewRelationMappingMaxCount); + } + for (RelationMapping queryToViewTableMapping : queryToViewTableMappings) { SlotMapping queryToViewSlotMapping = materializationContext.getSlotMappingFromCache(queryToViewTableMapping); @@ -650,7 +661,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac /** * Check the pattern of query or materializedView is supported or not. */ - protected boolean checkPattern(StructInfo structInfo) { + protected boolean checkPattern(StructInfo structInfo, CascadesContext cascadesContext) { if (structInfo.getRelations().isEmpty()) { return false; } @@ -676,7 +687,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac materializationId); if (cachedCheckResult == null) { // need check in real time - boolean checkResult = checkPattern(context.getStructInfo()); + boolean checkResult = checkPattern(context.getStructInfo(), cascadesContext); if (!checkResult) { context.recordFailReason(context.getStructInfo(), "View struct info is invalid", () -> String.format("view plan is %s", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java index b3e742c64c..d6d7817d35 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.rules.exploration.mv; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.trees.expressions.Alias; @@ -75,7 +76,7 @@ public abstract class MaterializedViewScanRule extends AbstractMaterializedViewR * Join condition should be slot reference equals currently. */ @Override - protected boolean checkPattern(StructInfo structInfo) { + protected boolean checkPattern(StructInfo structInfo, CascadesContext cascadesContext) { PlanCheckContext checkContext = PlanCheckContext.of(ImmutableSet.of()); return structInfo.getTopPlan().accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, checkContext) && !checkContext.isContainsTopAggregate(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 602c10249d..b17d6b3a41 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -529,6 +529,9 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_MATERIALIZED_VIEW_NEST_REWRITE = "enable_materialized_view_nest_rewrite"; + public static final String MATERIALIZED_VIEW_RELATION_MAPPING_MAX_COUNT + = "materialized_view_relation_mapping_max_count"; + public static final String CREATE_TABLE_PARTITION_MAX_NUM = "create_table_partition_max_num"; @@ -1660,6 +1663,12 @@ public class SessionVariable implements Serializable, Writable { "The max candidate num which participate in CBO when using asynchronous materialized views"}) public int materializedViewRewriteSuccessCandidateNum = 3; + @VariableMgr.VarAttr(name = MATERIALIZED_VIEW_RELATION_MAPPING_MAX_COUNT, needForward = true, + description = {"透明改写过程中，relation mapping最大允许数量，如果超过，进行截取", + "During transparent rewriting, relation mapping specifies the maximum allowed number. " + + "If the number exceeds the allowed number, the number is intercepted"}) + public int materializedViewRelationMappingMaxCount = 8; + @VariableMgr.VarAttr(name = ENABLE_MATERIALIZED_VIEW_UNION_REWRITE, needForward = true, description = {"当物化视图不足以提供查询的全部数据时，是否允许基表和物化视图 union 来响应查询", "When the materialized view is not enough to provide all the data for the query, " @@ -3730,6 +3739,10 @@ public class SessionVariable implements Serializable, Writable { return enableMaterializedViewNestRewrite; } + public int getMaterializedViewRelationMappingMaxCount() { + return materializedViewRelationMappingMaxCount; + } + public int getCreateTablePartitionMaxNum() { return createTablePartitionMaxNum; } diff --git a/regression-test/data/nereids_rules_p0/mv/availability/materialized_view_switch.out b/regression-test/data/nereids_rules_p0/mv/availability/materialized_view_switch.out new file mode 100644 index 0000000000..3e413a21d8 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/availability/materialized_view_switch.out @@ -0,0 +1,29 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query1_0_before -- +4 1 +4 2 +4 3 +4 4 +6 5 + +-- !query1_0_after -- +4 1 +4 2 +4 3 +4 4 +6 5 + +-- !query1_1_before -- +4 1 +4 2 +4 3 +4 4 +6 5 + +-- !query1_1_after -- +4 1 +4 2 +4 3 +4 4 +6 5 + diff --git a/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy b/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy index 29bd112e76..b4ae8b7bfc 100644 --- a/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy @@ -152,4 +152,42 @@ suite("materialized_view_switch") { sql "SET enable_materialized_view_rewrite=true" check_mv_rewrite_success(db, mv_name, query, "mv_name") sql """ DROP MATERIALIZED VIEW IF EXISTS mv_name""" + + // test when materialized_view_relation_mapping_max_count is 8 + def mv1_0 = """ + select t1.L_LINENUMBER, t2.l_extendedprice, t2.L_ORDERKEY + from lineitem t1 + inner join lineitem t2 on t1.L_ORDERKEY = t2.L_ORDERKEY; + """ + def query1_0 = """ + select t1.L_LINENUMBER, t2.L_ORDERKEY + from lineitem t1 + inner join lineitem t2 on t1.L_ORDERKEY = t2.L_ORDERKEY; + """ + order_qt_query1_0_before "${query1_0}" + check_mv_rewrite_success(db, mv1_0, query1_0, "mv1_0") + order_qt_query1_0_after "${query1_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" + + + // test when materialized_view_relation_mapping_max_count is 0 + sql "SET materialized_view_relation_mapping_max_count = 0" + + def mv1_1 = """ + select t1.L_LINENUMBER,t2.l_extendedprice, t2.L_ORDERKEY + from lineitem t1 + inner join lineitem t2 on t1.L_ORDERKEY = t2.L_ORDERKEY; + """ + def query1_1 = """ + select t1.L_LINENUMBER, t2.L_ORDERKEY + from lineitem t1 + inner join lineitem t2 on t1.L_ORDERKEY = t2.L_ORDERKEY; + """ + order_qt_query1_1_before "${query1_1}" + check_mv_rewrite_fail(db, mv1_1, query1_1, "mv1_1") + order_qt_query1_1_after "${query1_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_1""" + + sql "SET materialized_view_relation_mapping_max_count = 8" + } \ No newline at end of file