[opt](mtmv) Add threshold for relation mapping num when query rewrite (#34694) (#35378)

if query and mv def is as following: def mv1_1 = """ select t1.L_LINENUMBER,t2.l_extendedprice, t2.L_ORDERKEY from lineitem t1 inner join lineitem t2 on t1.L_ORDERKEY = t2.L_ORDERKEY; """ def query1_1 = """ select t1.L_LINENUMBER, t2.L_ORDERKEY from lineitem t1 inner join lineitem t2 on t1.L_ORDERKEY = t2.L_ORDERKEY; """ this will generate relation mapping by Cartesian, if the num of self join is too much, this will cause the performance problem so we add `materialized_view_relation_mapping_max_count` session varaible, default 8. if actual num is greater than the value, the excess relation mapping is discarded.
2024-05-24 20:36:29 +08:00
parent 0f550aeda7
commit 62998719df
7 changed files with 100 additions and 6 deletions
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
@ -18,6 +18,7 @@
 package org.apache.doris.nereids.rules.exploration.mv;

 import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.CascadesContext;
 import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext;
 import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanSplitContext;
 import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
@ -370,7 +371,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
     * slot reference equals currently.
     */
    @Override
-    protected boolean checkPattern(StructInfo structInfo) {
+    protected boolean checkPattern(StructInfo structInfo, CascadesContext cascadesContext) {
        PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET);
        // if query or mv contains more then one top aggregate, should fail
        return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext)
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java
@ -17,6 +17,7 @@

 package org.apache.doris.nereids.rules.exploration.mv;

+import org.apache.doris.nereids.CascadesContext;
 import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext;
 import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
 import org.apache.doris.nereids.trees.expressions.Alias;
@ -74,7 +75,7 @@ public abstract class AbstractMaterializedViewJoinRule extends AbstractMateriali
     * Join condition should be slot reference equals currently.
     */
    @Override
-    protected boolean checkPattern(StructInfo structInfo) {
+    protected boolean checkPattern(StructInfo structInfo, CascadesContext cascadesContext) {
        PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET);
        return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext)
                && !checkContext.isContainsTopAggregate();
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
@ -66,6 +66,8 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Sets;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;

 import java.util.ArrayList;
 import java.util.BitSet;
@ -82,6 +84,8 @@ import java.util.stream.Collectors;
 * The abstract class for all materialized view rules
 */
 public abstract class AbstractMaterializedViewRule implements ExplorationRuleFactory {
+
+    public static final Logger LOG = LogManager.getLogger(AbstractMaterializedViewRule.class);
    public static final Set<JoinType> SUPPORTED_JOIN_TYPE_SET = ImmutableSet.of(
            JoinType.INNER_JOIN,
            JoinType.LEFT_OUTER_JOIN,
@ -142,7 +146,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
        List<StructInfo> uncheckedStructInfos = MaterializedViewUtils.extractStructInfo(queryPlan, cascadesContext,
                materializedViewTableSet);
        uncheckedStructInfos.forEach(queryStructInfo -> {
-            boolean valid = checkPattern(queryStructInfo) && queryStructInfo.isValid();
+            boolean valid = checkPattern(queryStructInfo, cascadesContext) && queryStructInfo.isValid();
            if (!valid) {
                cascadesContext.getMaterializationContexts().forEach(ctx ->
                        ctx.recordFailReason(queryStructInfo, "Query struct info is invalid",
@ -178,6 +182,13 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
                    "Query to view table mapping is null", () -> "");
            return rewriteResults;
        }
+        int materializedViewRelationMappingMaxCount = cascadesContext.getConnectContext().getSessionVariable()
+                .getMaterializedViewRelationMappingMaxCount();
+        if (queryToViewTableMappings.size() > materializedViewRelationMappingMaxCount) {
+            LOG.warn("queryToViewTableMappings is over limit and be intercepted");
+            queryToViewTableMappings = queryToViewTableMappings.subList(0, materializedViewRelationMappingMaxCount);
+        }
+
        for (RelationMapping queryToViewTableMapping : queryToViewTableMappings) {
            SlotMapping queryToViewSlotMapping =
                    materializationContext.getSlotMappingFromCache(queryToViewTableMapping);
@ -650,7 +661,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
    /**
     * Check the pattern of query or materializedView is supported or not.
     */
-    protected boolean checkPattern(StructInfo structInfo) {
+    protected boolean checkPattern(StructInfo structInfo, CascadesContext cascadesContext) {
        if (structInfo.getRelations().isEmpty()) {
            return false;
        }
@ -676,7 +687,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
                materializationId);
        if (cachedCheckResult == null) {
            // need check in real time
-            boolean checkResult = checkPattern(context.getStructInfo());
+            boolean checkResult = checkPattern(context.getStructInfo(), cascadesContext);
            if (!checkResult) {
                context.recordFailReason(context.getStructInfo(),
                        "View struct info is invalid", () -> String.format("view plan is %s",
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java
@ -17,6 +17,7 @@

 package org.apache.doris.nereids.rules.exploration.mv;

+import org.apache.doris.nereids.CascadesContext;
 import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext;
 import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
 import org.apache.doris.nereids.trees.expressions.Alias;
@ -75,7 +76,7 @@ public abstract class MaterializedViewScanRule extends AbstractMaterializedViewR
     * Join condition should be slot reference equals currently.
     */
    @Override
-    protected boolean checkPattern(StructInfo structInfo) {
+    protected boolean checkPattern(StructInfo structInfo, CascadesContext cascadesContext) {
        PlanCheckContext checkContext = PlanCheckContext.of(ImmutableSet.of());
        return structInfo.getTopPlan().accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, checkContext)
                && !checkContext.isContainsTopAggregate();
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@ -529,6 +529,9 @@ public class SessionVariable implements Serializable, Writable {
    public static final String ENABLE_MATERIALIZED_VIEW_NEST_REWRITE
            = "enable_materialized_view_nest_rewrite";

+    public static final String MATERIALIZED_VIEW_RELATION_MAPPING_MAX_COUNT
+            = "materialized_view_relation_mapping_max_count";
+
    public static final String CREATE_TABLE_PARTITION_MAX_NUM
            = "create_table_partition_max_num";

@ -1660,6 +1663,12 @@ public class SessionVariable implements Serializable, Writable {
                    "The max candidate num which participate in CBO when using asynchronous materialized views"})
    public int materializedViewRewriteSuccessCandidateNum = 3;

+    @VariableMgr.VarAttr(name = MATERIALIZED_VIEW_RELATION_MAPPING_MAX_COUNT, needForward = true,
+            description = {"透明改写过程中，relation mapping最大允许数量，如果超过，进行截取",
+                    "During transparent rewriting, relation mapping specifies the maximum allowed number. "
+                            + "If the number exceeds the allowed number, the number is intercepted"})
+    public int materializedViewRelationMappingMaxCount = 8;
+
    @VariableMgr.VarAttr(name = ENABLE_MATERIALIZED_VIEW_UNION_REWRITE, needForward = true,
            description = {"当物化视图不足以提供查询的全部数据时，是否允许基表和物化视图 union 来响应查询",
                    "When the materialized view is not enough to provide all the data for the query, "
@ -3730,6 +3739,10 @@ public class SessionVariable implements Serializable, Writable {
        return enableMaterializedViewNestRewrite;
    }

+    public int getMaterializedViewRelationMappingMaxCount() {
+        return materializedViewRelationMappingMaxCount;
+    }
+
    public int getCreateTablePartitionMaxNum() {
        return createTablePartitionMaxNum;
    }