From 9a58cacf0f5062cc2c0b535a794519fecb52ce73 Mon Sep 17 00:00:00 2001 From: seawinde <149132972+seawinde@users.noreply.github.com> Date: Mon, 22 Jan 2024 13:04:04 +0800 Subject: [PATCH] [Improvement](nereids) Make sure to catch and record exception for every materialization context (#29953) 1. Make sure instance when change params of StructInfo,Predicates. 2. Catch and record exception for every materialization context, this make sure that if throw exception when one materialization context rewrite, it will not influence others. 3. Support to mv rewrite when hava count function when aggregate without group by --- .../java/org/apache/doris/mtmv/MTMVCache.java | 4 +- ...AbstractMaterializedViewAggregateRule.java | 114 ++---- .../mv/AbstractMaterializedViewJoinRule.java | 17 +- .../mv/AbstractMaterializedViewRule.java | 378 ++++++++++-------- .../mv/MaterializedViewAggregateRule.java | 2 +- .../rules/exploration/mv/Predicates.java | 28 +- .../rules/exploration/mv/StructInfo.java | 147 ++++--- .../rules/rewrite/NormalizeToSlot.java | 23 +- .../apache/doris/nereids/trees/TreeNode.java | 17 - .../trees/plans/logical/LogicalProject.java | 4 + .../doris/nereids/util/ExpressionUtils.java | 4 +- .../aggregate_with_roll_up.out | 6 + .../aggregate_with_roll_up.groovy | 131 +++--- 13 files changed, 486 insertions(+), 389 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java index 3d776d9a7a..a7ddeeb170 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java @@ -63,7 +63,9 @@ public class MTMVCache { StatementContext mvSqlStatementContext = new StatementContext(connectContext, new OriginStatement(mtmv.getQuerySql(), 0)); NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext); - + if (mvSqlStatementContext.getConnectContext().getStatementContext() == null) { + mvSqlStatementContext.getConnectContext().setStatementContext(mvSqlStatementContext); + } Plan mvRewrittenPlan = planner.plan(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN); Plan mvPlan = mvRewrittenPlan instanceof LogicalResultSink diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index e47e15dd56..48b900ca74 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -23,8 +23,8 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge; import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.AbstractNode; import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanSplitContext; -import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Any; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.ExprId; @@ -45,7 +45,6 @@ import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; @@ -64,7 +63,7 @@ import java.util.stream.Collectors; */ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMaterializedViewRule { - protected static final Multimap + protected static final Multimap AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP = ArrayListMultimap.create(); static { @@ -95,7 +94,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInfo, StructInfo viewStructInfo, - SlotMapping queryToViewSlotMapping, + SlotMapping viewToQuerySlotMapping, Plan tempRewritedPlan, MaterializationContext materializationContext) { // get view and query aggregate and top plan correspondingly @@ -115,12 +114,11 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate } // Firstly,if group by expression between query and view is equals, try to rewrite expression directly Plan queryTopPlan = queryTopPlanAndAggPair.key(); - SlotMapping viewToQurySlotMapping = queryToViewSlotMapping.inverse(); - if (isGroupByEquals(queryTopPlanAndAggPair, viewTopPlanAndAggPair, viewToQurySlotMapping)) { + if (isGroupByEquals(queryTopPlanAndAggPair, viewTopPlanAndAggPair, viewToQuerySlotMapping)) { List rewrittenQueryExpressions = rewriteExpression(queryTopPlan.getExpressions(), queryTopPlan, materializationContext.getMvExprToMvScanExprMapping(), - queryToViewSlotMapping, + viewToQuerySlotMapping, true); if (!rewrittenQueryExpressions.isEmpty()) { return new LogicalProject<>( @@ -133,15 +131,17 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), Pair.of("Can not rewrite expression when no roll up", String.format("expressionToWrite = %s,\n mvExprToMvScanExprMapping = %s,\n" - + "queryToViewSlotMapping = %s", + + "viewToQuerySlotMapping = %s", queryTopPlan.getExpressions(), materializationContext.getMvExprToMvScanExprMapping(), - queryToViewSlotMapping))); + viewToQuerySlotMapping))); } // if view is scalar aggregate but query is not. Or if query is scalar aggregate but view is not // Should not rewrite - if (queryTopPlanAndAggPair.value().getGroupByExpressions().isEmpty() - || viewTopPlanAndAggPair.value().getGroupByExpressions().isEmpty()) { + List queryGroupByExpressions = queryTopPlanAndAggPair.value().getGroupByExpressions(); + List viewGroupByExpressions = viewTopPlanAndAggPair.value().getGroupByExpressions(); + if ((queryGroupByExpressions.isEmpty() && !viewGroupByExpressions.isEmpty()) + || (!queryGroupByExpressions.isEmpty() && viewGroupByExpressions.isEmpty())) { materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), Pair.of("only one the of query or view is scalar aggregate and " + "can not rewrite expression meanwhile", @@ -154,53 +154,42 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate // split the query top plan expressions to group expressions and functions, if can not, bail out. Pair, Set> queryGroupAndFunctionPair = topPlanSplitToGroupAndFunction(queryTopPlanAndAggPair); - // this map will be used to rewrite expression - Multimap needRollupExprMap = HashMultimap.create(); - Multimap groupRewrittenExprMap = HashMultimap.create(); - // permute the mv expr mapping to query based - Map mvExprToMvScanExprQueryBased = - materializationContext.getMvExprToMvScanExprMapping().keyPermute(viewToQurySlotMapping) - .flattenMap().get(0); Set queryTopPlanFunctionSet = queryGroupAndFunctionPair.value(); // try to rewrite, contains both roll up aggregate functions and aggregate group expression List finalAggregateExpressions = new ArrayList<>(); List finalGroupExpressions = new ArrayList<>(); - for (Expression topExpression : queryTopPlan.getExpressions()) { + List queryExpressions = queryTopPlan.getExpressions(); + // permute the mv expr mapping to query based + Map mvExprToMvScanExprQueryBased = + materializationContext.getMvExprToMvScanExprMapping().keyPermute(viewToQuerySlotMapping) + .flattenMap().get(0); + for (Expression topExpression : queryExpressions) { // if agg function, try to roll up and rewrite if (queryTopPlanFunctionSet.contains(topExpression)) { Expression queryFunctionShuttled = ExpressionUtils.shuttleExpressionWithLineage( topExpression, queryTopPlan); // try to roll up - AggregateFunction queryFunction = (AggregateFunction) queryFunctionShuttled.firstMatch( - expr -> expr instanceof AggregateFunction); - Function rollupAggregateFunction = rollup(queryFunction, queryFunctionShuttled, - mvExprToMvScanExprQueryBased); + List queryFunctions = + queryFunctionShuttled.collectFirst(expr -> expr instanceof AggregateFunction); + if (queryFunctions.isEmpty()) { + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + Pair.of("Can not found query function", + String.format("queryFunctionShuttled = %s", queryFunctionShuttled))); + return null; + } + Function rollupAggregateFunction = rollup((AggregateFunction) queryFunctions.get(0), + queryFunctionShuttled, mvExprToMvScanExprQueryBased); if (rollupAggregateFunction == null) { materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), Pair.of("Query function roll up fail", String.format("queryFunction = %s,\n queryFunctionShuttled = %s,\n" + "mvExprToMvScanExprQueryBased = %s", - queryFunction, queryFunctionShuttled, mvExprToMvScanExprQueryBased))); + queryFunctions.get(0), queryFunctionShuttled, + mvExprToMvScanExprQueryBased))); return null; } - // key is query need roll up expr, value is mv scan based roll up expr - needRollupExprMap.put(queryFunctionShuttled, rollupAggregateFunction); - // rewrite query function expression by mv expression - ExpressionMapping needRollupExprMapping = new ExpressionMapping(needRollupExprMap); - Expression rewrittenFunctionExpression = rewriteExpression(topExpression, - queryTopPlan, - needRollupExprMapping, - queryToViewSlotMapping, - false); - if (rewrittenFunctionExpression == null) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Roll up expression can not rewrite by view", String.format( - "topExpression = %s,\n needRollupExprMapping = %s,\n queryToViewSlotMapping = %s", - topExpression, needRollupExprMapping, queryToViewSlotMapping))); - return null; - } - finalAggregateExpressions.add((NamedExpression) rewrittenFunctionExpression); + finalAggregateExpressions.add(new Alias(rollupAggregateFunction)); } else { // if group by expression, try to rewrite group by expression Expression queryGroupShuttledExpr = @@ -213,26 +202,9 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate mvExprToMvScanExprQueryBased, queryGroupShuttledExpr))); return null; } - groupRewrittenExprMap.put(queryGroupShuttledExpr, - mvExprToMvScanExprQueryBased.get(queryGroupShuttledExpr)); - // rewrite query group expression by mv expression - ExpressionMapping groupRewrittenExprMapping = new ExpressionMapping(groupRewrittenExprMap); - Expression rewrittenGroupExpression = rewriteExpression( - topExpression, - queryTopPlan, - groupRewrittenExprMapping, - queryToViewSlotMapping, - true); - if (rewrittenGroupExpression == null) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Query dimensions can not be rewritten by view", - String.format("topExpression is %s,\n groupRewrittenExprMapping is %s,\n" - + "queryToViewSlotMapping = %s", - topExpression, groupRewrittenExprMapping, queryToViewSlotMapping))); - return null; - } - finalAggregateExpressions.add((NamedExpression) rewrittenGroupExpression); - finalGroupExpressions.add(rewrittenGroupExpression); + Expression expression = mvExprToMvScanExprQueryBased.get(queryGroupShuttledExpr); + finalAggregateExpressions.add((NamedExpression) expression); + finalGroupExpressions.add(expression); } } // add project to guarantee group by column ref is slot reference, @@ -271,7 +243,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate private boolean isGroupByEquals(Pair> queryTopPlanAndAggPair, Pair> viewTopPlanAndAggPair, - SlotMapping viewToQurySlotMapping) { + SlotMapping viewToQuerySlotMapping) { Plan queryTopPlan = queryTopPlanAndAggPair.key(); Plan viewTopPlan = viewTopPlanAndAggPair.key(); LogicalAggregate queryAggregate = queryTopPlanAndAggPair.value(); @@ -282,7 +254,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate Set viewGroupShuttledExpressionQueryBased = ExpressionUtils.shuttleExpressionWithLineage( viewAggregate.getGroupByExpressions(), viewTopPlan) .stream() - .map(expr -> ExpressionUtils.replace(expr, viewToQurySlotMapping.toSlotReferenceMap())) + .map(expr -> ExpressionUtils.replace(expr, viewToQuerySlotMapping.toSlotReferenceMap())) .collect(Collectors.toSet()); return queryGroupShuttledExpression.equals(viewGroupShuttledExpressionQueryBased); } @@ -309,19 +281,20 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate } Expression rollupParam = null; Expression viewRollupFunction = null; - if (mvExprToMvScanExprQueryBased.containsKey(queryAggregateFunctionShuttled)) { - // function can rewrite by view + // handle simple aggregate function roll up which is not in the AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP + if (mvExprToMvScanExprQueryBased.containsKey(queryAggregateFunctionShuttled) + && AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.keySet().stream() + .noneMatch(aggFunction -> aggFunction.equals(queryAggregateFunction))) { rollupParam = mvExprToMvScanExprQueryBased.get(queryAggregateFunctionShuttled); viewRollupFunction = queryAggregateFunctionShuttled; } else { - // function can not rewrite by view, try to use complex roll up param + // handle complex functions roll up // eg: query is count(distinct param), mv sql is bitmap_union(to_bitmap(param)) for (Expression mvExprShuttled : mvExprToMvScanExprQueryBased.keySet()) { if (!(mvExprShuttled instanceof Function)) { continue; } - if (isAggregateFunctionEquivalent(queryAggregateFunction, queryAggregateFunctionShuttled, - (Function) mvExprShuttled)) { + if (isAggregateFunctionEquivalent(queryAggregateFunction, (Function) mvExprShuttled)) { rollupParam = mvExprToMvScanExprQueryBased.get(mvExprShuttled); viewRollupFunction = mvExprShuttled; } @@ -429,13 +402,12 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate * This will check the count(distinct a) in query is equivalent to bitmap_union(to_bitmap(a)) in mv, * and then check their arguments is equivalent. */ - private boolean isAggregateFunctionEquivalent(Function queryFunction, Expression queryFunctionShuttled, - Function viewFunction) { + private boolean isAggregateFunctionEquivalent(Function queryFunction, Function viewFunction) { if (queryFunction.equals(viewFunction)) { return true; } // check the argument of rollup function is equivalent to view function or not - for (Map.Entry> equivalentFunctionEntry : + for (Map.Entry> equivalentFunctionEntry : AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.asMap().entrySet()) { if (equivalentFunctionEntry.getKey().equals(queryFunction)) { // check is have equivalent function or not diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java index a482b13b5e..57894ac17c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge; import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.AbstractNode; import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; @@ -47,7 +48,7 @@ public abstract class AbstractMaterializedViewJoinRule extends AbstractMateriali protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInfo, StructInfo viewStructInfo, - SlotMapping queryToViewSlotMapping, + SlotMapping targetToSourceMapping, Plan tempRewritedPlan, MaterializationContext materializationContext) { // Rewrite top projects, represent the query projects by view @@ -55,19 +56,18 @@ public abstract class AbstractMaterializedViewJoinRule extends AbstractMateriali queryStructInfo.getExpressions(), queryStructInfo.getOriginalPlan(), materializationContext.getMvExprToMvScanExprMapping(), - queryToViewSlotMapping, + targetToSourceMapping, true ); // Can not rewrite, bail out - if (expressionsRewritten.isEmpty() - || expressionsRewritten.stream().anyMatch(expr -> !(expr instanceof NamedExpression))) { + if (expressionsRewritten.isEmpty()) { materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), Pair.of("Rewrite expressions by view in join fail", String.format("expressionToRewritten is %s,\n mvExprToMvScanExprMapping is %s,\n" - + "queryToViewSlotMapping = %s", + + "targetToSourceMapping = %s", queryStructInfo.getExpressions(), materializationContext.getMvExprToMvScanExprMapping(), - queryToViewSlotMapping))); + targetToSourceMapping))); return null; } // record the group id in materializationContext, and when rewrite again in @@ -77,7 +77,10 @@ public abstract class AbstractMaterializedViewJoinRule extends AbstractMateriali queryStructInfo.getOriginalPlan().getGroupExpression().get().getOwnerGroup().getGroupId()); } return new LogicalProject<>( - expressionsRewritten.stream().map(NamedExpression.class::cast).collect(Collectors.toList()), + expressionsRewritten.stream() + .map(expression -> expression instanceof NamedExpression ? expression : new Alias(expression)) + .map(NamedExpression.class::cast) + .collect(Collectors.toList()), tempRewritedPlan); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 352bccc019..ea035769b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -48,6 +48,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.TypeUtils; @@ -72,163 +73,222 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac JoinType.LEFT_OUTER_JOIN); /** - * The abstract template method for query rewrite, it contains the main logic and different query - * pattern should override the sub logic. + * The abstract template method for query rewrite, it contains the main logic, try to rewrite query by + * multi materialization every time. if exception it will catch the exception and record it to + * materialization context. */ - protected List rewrite(Plan queryPlan, CascadesContext cascadesContext) { + public List rewrite(Plan queryPlan, CascadesContext cascadesContext) { + List rewrittenPlans = new ArrayList<>(); + // already rewrite or query is invalid, bail out + List queryStructInfos = checkQuery(queryPlan, cascadesContext); + if (queryStructInfos.isEmpty()) { + return rewrittenPlans; + } + for (MaterializationContext context : cascadesContext.getMaterializationContexts()) { + if (checkIfRewritten(queryPlan, context)) { + continue; + } + // TODO Just support only one query struct info, support multi later. + StructInfo queryStructInfo = queryStructInfos.get(0); + try { + rewrittenPlans.addAll(doRewrite(queryStructInfo, cascadesContext, context)); + } catch (Exception exception) { + context.recordFailReason(queryStructInfo.getOriginalPlanId(), + Pair.of("Materialized view rule exec fail", exception.toString())); + } + } + return rewrittenPlans; + } + + /** + * Check query is valid or not, if valid return the query struct infos, if invalid return empty list. + */ + protected List checkQuery(Plan queryPlan, CascadesContext cascadesContext) { + List validQueryStructInfos = new ArrayList<>(); List materializationContexts = cascadesContext.getMaterializationContexts(); - List rewriteResults = new ArrayList<>(); if (materializationContexts.isEmpty()) { - return rewriteResults; + return validQueryStructInfos; } List queryStructInfos = MaterializedViewUtils.extractStructInfo(queryPlan, cascadesContext); // TODO Just Check query queryPlan firstly, support multi later. StructInfo queryStructInfo = queryStructInfos.get(0); if (!checkPattern(queryStructInfo)) { - materializationContexts.forEach(ctx -> ctx.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Query struct info is invalid", - String.format("queryPlan is %s", queryPlan.treeString())))); + cascadesContext.getMaterializationContexts().forEach(ctx -> + ctx.recordFailReason(queryStructInfo.getOriginalPlanId(), + Pair.of("Query struct info is invalid", + String.format("queryPlan is %s", queryPlan.treeString()))) + ); + return validQueryStructInfos; + } + validQueryStructInfos.add(queryStructInfo); + return validQueryStructInfos; + } + + /** + * The abstract template method for query rewrite, it contains the main logic, try to rewrite query by + * only one materialization every time. Different query pattern should override the sub logic. + */ + protected List doRewrite(StructInfo queryStructInfo, CascadesContext cascadesContext, + MaterializationContext materializationContext) { + List rewriteResults = new ArrayList<>(); + List viewStructInfos = MaterializedViewUtils.extractStructInfo( + materializationContext.getMvPlan(), cascadesContext); + if (viewStructInfos.size() > 1) { + // view struct info should only have one + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + Pair.of("The num of view struct info is more then one", + String.format("mv plan is %s", materializationContext.getMvPlan().treeString()))); return rewriteResults; } - for (MaterializationContext materializationContext : materializationContexts) { - // already rewrite, bail out - if (checkIfRewritten(queryPlan, materializationContext)) { + StructInfo viewStructInfo = viewStructInfos.get(0); + if (!checkPattern(viewStructInfo)) { + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + Pair.of("View struct info is invalid", + String.format(", view plan is %s", viewStructInfo.getOriginalPlan().treeString()))); + return rewriteResults; + } + MatchMode matchMode = decideMatchMode(queryStructInfo.getRelations(), viewStructInfo.getRelations()); + if (MatchMode.COMPLETE != matchMode) { + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + Pair.of("Match mode is invalid", String.format("matchMode is %s", matchMode))); + return rewriteResults; + } + List queryToViewTableMappings = RelationMapping.generate(queryStructInfo.getRelations(), + viewStructInfo.getRelations()); + // if any relation in query and view can not map, bail out. + if (queryToViewTableMappings == null) { + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + Pair.of("Query to view table mapping is null", "")); + return rewriteResults; + } + for (RelationMapping queryToViewTableMapping : queryToViewTableMappings) { + SlotMapping queryToViewSlotMapping = SlotMapping.generate(queryToViewTableMapping); + if (queryToViewSlotMapping == null) { + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + Pair.of("Query to view slot mapping is null", "")); continue; } - List viewStructInfos = MaterializedViewUtils.extractStructInfo( - materializationContext.getMvPlan(), cascadesContext); - if (viewStructInfos.size() > 1) { - // view struct info should only have one + SlotMapping viewToQuerySlotMapping = queryToViewSlotMapping.inverse(); + LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from( + queryToViewTableMapping, queryToViewSlotMapping, queryStructInfo, viewStructInfo); + ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo, + compatibilityContext); + if (comparisonResult.isInvalid()) { materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("The num of view struct info is more then one", - String.format("mv plan is %s", materializationContext.getMvPlan().treeString()))); - return rewriteResults; - } - StructInfo viewStructInfo = viewStructInfos.get(0); - if (!checkPattern(viewStructInfo)) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("View struct info is invalid", - String.format(", view plan is %s", viewStructInfo.getOriginalPlan().treeString()))); + Pair.of("The graph logic between query and view is not consistent", + comparisonResult.getErrorMessage())); continue; } - MatchMode matchMode = decideMatchMode(queryStructInfo.getRelations(), viewStructInfo.getRelations()); - if (MatchMode.COMPLETE != matchMode) { + SplitPredicate compensatePredicates = predicatesCompensate(queryStructInfo, viewStructInfo, + viewToQuerySlotMapping, comparisonResult, cascadesContext); + // Can not compensate, bail out + if (compensatePredicates.isInvalid()) { materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Match mode is invalid", String.format("matchMode is %s", matchMode))); + Pair.of("Predicate compensate fail", + String.format("query predicates = %s,\n query equivalenceClass = %s, \n" + + "view predicates = %s,\n query equivalenceClass = %s\n", + queryStructInfo.getPredicates(), + queryStructInfo.getEquivalenceClass(), + viewStructInfo.getPredicates(), + viewStructInfo.getEquivalenceClass()))); continue; } - List queryToViewTableMappings = RelationMapping.generate(queryStructInfo.getRelations(), - viewStructInfo.getRelations()); - // if any relation in query and view can not map, bail out. - if (queryToViewTableMappings == null) { + Plan rewrittenPlan; + Plan mvScan = materializationContext.getMvScanPlan(); + Plan originalPlan = queryStructInfo.getOriginalPlan(); + if (compensatePredicates.isAlwaysTrue()) { + rewrittenPlan = mvScan; + } else { + // Try to rewrite compensate predicates by using mv scan + List rewriteCompensatePredicates = rewriteExpression(compensatePredicates.toList(), + originalPlan, materializationContext.getMvExprToMvScanExprMapping(), + viewToQuerySlotMapping, true); + if (rewriteCompensatePredicates.isEmpty()) { + materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), + Pair.of("Rewrite compensate predicate by view fail", String.format( + "compensatePredicates = %s,\n mvExprToMvScanExprMapping = %s,\n" + + "viewToQuerySlotMapping = %s", + compensatePredicates, + materializationContext.getMvExprToMvScanExprMapping(), + viewToQuerySlotMapping))); + continue; + } + rewrittenPlan = new LogicalFilter<>(Sets.newHashSet(rewriteCompensatePredicates), mvScan); + } + // Rewrite query by view + rewrittenPlan = rewriteQueryByView(matchMode, queryStructInfo, viewStructInfo, viewToQuerySlotMapping, + rewrittenPlan, materializationContext); + if (rewrittenPlan == null) { + continue; + } + rewrittenPlan = rewriteByRules(cascadesContext, rewrittenPlan, originalPlan); + if (!isOutputValid(originalPlan, rewrittenPlan)) { + ObjectId planObjId = originalPlan.getGroupExpression().map(GroupExpression::getId) + .orElseGet(() -> new ObjectId(-1)); + materializationContext.recordFailReason(planObjId, Pair.of( + "RewrittenPlan output logical properties is different with target group", + String.format("planOutput logical properties = %s,\n" + + "groupOutput logical properties = %s", rewrittenPlan.getLogicalProperties(), + originalPlan.getLogicalProperties()))); + continue; + } + // check the partitions used by rewritten plan is valid or not + Set invalidPartitionsQueryUsed = + calcInvalidPartitions(rewrittenPlan, materializationContext, cascadesContext); + if (!invalidPartitionsQueryUsed.isEmpty()) { materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Query to view table mapping is null", "")); - return rewriteResults; - } - for (RelationMapping queryToViewTableMapping : queryToViewTableMappings) { - SlotMapping queryToViewSlotMapping = SlotMapping.generate(queryToViewTableMapping); - if (queryToViewSlotMapping == null) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Query to view slot mapping is null", "")); - continue; - } - LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from( - queryToViewTableMapping, queryToViewSlotMapping, queryStructInfo, viewStructInfo); - ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo, - compatibilityContext); - if (comparisonResult.isInvalid()) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("The graph logic between query and view is not consistent", - comparisonResult.getErrorMessage())); - continue; - } - SplitPredicate compensatePredicates = predicatesCompensate(queryStructInfo, viewStructInfo, - queryToViewSlotMapping, comparisonResult, cascadesContext); - // Can not compensate, bail out - if (compensatePredicates.isInvalid()) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Predicate compensate fail", - String.format("query predicates = %s,\n query equivalenceClass = %s, \n" - + "view predicates = %s,\n query equivalenceClass = %s\n", - queryStructInfo.getPredicates(), - queryStructInfo.getEquivalenceClass(), - viewStructInfo.getPredicates(), - viewStructInfo.getEquivalenceClass()))); - continue; - } - Plan rewrittenPlan; - Plan mvScan = materializationContext.getMvScanPlan(); - if (compensatePredicates.isAlwaysTrue()) { - rewrittenPlan = mvScan; - } else { - // Try to rewrite compensate predicates by using mv scan - List rewriteCompensatePredicates = rewriteExpression(compensatePredicates.toList(), - queryPlan, materializationContext.getMvExprToMvScanExprMapping(), queryToViewSlotMapping, - true); - if (rewriteCompensatePredicates.isEmpty()) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Rewrite compensate predicate by view fail", String.format( - "compensatePredicates = %s,\n mvExprToMvScanExprMapping = %s,\n" - + "queryToViewSlotMapping = %s", - compensatePredicates, - materializationContext.getMvExprToMvScanExprMapping(), - queryToViewSlotMapping))); - continue; - } - rewrittenPlan = new LogicalFilter<>(Sets.newHashSet(rewriteCompensatePredicates), mvScan); - } - // Rewrite query by view - rewrittenPlan = rewriteQueryByView(matchMode, queryStructInfo, viewStructInfo, queryToViewSlotMapping, - rewrittenPlan, materializationContext); - if (rewrittenPlan == null) { - continue; - } - // run rbo job on mv rewritten plan - CascadesContext rewrittenPlanContext = CascadesContext.initContext( - cascadesContext.getStatementContext(), rewrittenPlan, - cascadesContext.getCurrentJobContext().getRequiredProperties()); - Rewriter.getWholeTreeRewriter(rewrittenPlanContext).execute(); - rewrittenPlan = rewrittenPlanContext.getRewritePlan(); - if (!checkOutput(queryPlan, rewrittenPlan, materializationContext)) { - continue; - } - // check the partitions used by rewritten plan is valid or not - Set invalidPartitionsQueryUsed = - calcInvalidPartitions(rewrittenPlan, materializationContext, cascadesContext); - if (!invalidPartitionsQueryUsed.isEmpty()) { - materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(), - Pair.of("Check partition query used validation fail", - String.format("the partition used by query is invalid by materialized view," - + "invalid partition info query used is %s", - materializationContext.getMTMV().getPartitions().stream() - .filter(partition -> - invalidPartitionsQueryUsed.contains(partition.getId())) - .collect(Collectors.toSet())))); - continue; - } - recordIfRewritten(queryPlan, materializationContext); - rewriteResults.add(rewrittenPlan); + Pair.of("Check partition query used validation fail", + String.format("the partition used by query is invalid by materialized view," + + "invalid partition info query used is %s", + materializationContext.getMTMV().getPartitions().stream() + .filter(partition -> + invalidPartitionsQueryUsed.contains(partition.getId())) + .collect(Collectors.toSet())))); + continue; } + recordIfRewritten(originalPlan, materializationContext); + rewriteResults.add(rewrittenPlan); } return rewriteResults; } /** - * Check the logical properties of rewritten plan by mv is the same with source plan + * Rewrite by rules and try to make output is the same after optimize by rules */ - protected boolean checkOutput(Plan sourcePlan, Plan rewrittenPlan, MaterializationContext materializationContext) { + protected Plan rewriteByRules(CascadesContext cascadesContext, Plan rewrittenPlan, Plan originPlan) { + // run rbo job on mv rewritten plan + CascadesContext rewrittenPlanContext = CascadesContext.initContext( + cascadesContext.getStatementContext(), rewrittenPlan, + cascadesContext.getCurrentJobContext().getRequiredProperties()); + Rewriter.getWholeTreeRewriter(rewrittenPlanContext).execute(); + rewrittenPlan = rewrittenPlanContext.getRewritePlan(); + List originPlanOutput = originPlan.getOutput(); + List rewrittenPlanOutput = rewrittenPlan.getOutput(); + if (originPlanOutput.size() != rewrittenPlanOutput.size()) { + return null; + } + List expressions = new ArrayList<>(); + // should add project above rewritten plan if top plan is not project, if aggregate above will nu + if (!isOutputValid(originPlan, rewrittenPlan)) { + for (int i = 0; i < originPlanOutput.size(); i++) { + expressions.add(((NamedExpression) normalizeExpression(originPlanOutput.get(i), + rewrittenPlanOutput.get(i)))); + } + return new LogicalProject<>(expressions, rewrittenPlan, false); + } + return rewrittenPlan; + } + + /** + * Check the logical properties of rewritten plan by mv is the same with source plan + * if same return true, if different return false + */ + protected boolean isOutputValid(Plan sourcePlan, Plan rewrittenPlan) { if (sourcePlan.getGroupExpression().isPresent() && !rewrittenPlan.getLogicalProperties() .equals(sourcePlan.getGroupExpression().get().getOwnerGroup().getLogicalProperties())) { - ObjectId planObjId = sourcePlan.getGroupExpression().map(GroupExpression::getId) - .orElseGet(() -> new ObjectId(-1)); - materializationContext.recordFailReason(planObjId, Pair.of( - "RewrittenPlan output logical properties is different with target group", - String.format("planOutput logical properties = %s,\n" - + "groupOutput logical properties = %s", rewrittenPlan.getLogicalProperties(), - sourcePlan.getGroupExpression().get().getOwnerGroup().getLogicalProperties()))); return false; } - return true; + return sourcePlan.getLogicalProperties().equals(rewrittenPlan.getLogicalProperties()); } /** @@ -271,7 +331,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac * Rewrite query by view, for aggregate or join rewriting should be different inherit class implementation */ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInfo, StructInfo viewStructInfo, - SlotMapping queryToViewSlotMapping, Plan tempRewritedPlan, MaterializationContext materializationContext) { + SlotMapping viewToQuerySlotMapping, Plan tempRewritedPlan, MaterializationContext materializationContext) { return tempRewritedPlan; } @@ -298,7 +358,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac * target */ protected List rewriteExpression(List sourceExpressionsToWrite, Plan sourcePlan, - ExpressionMapping targetExpressionMapping, SlotMapping sourceToTargetMapping, + ExpressionMapping targetExpressionMapping, SlotMapping targetToSourceMapping, boolean targetExpressionNeedSourceBased) { // Firstly, rewrite the target expression using source with inverse mapping // then try to use the target expression to represent the query. if any of source expressions @@ -307,14 +367,13 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac List sourceShuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage( sourceExpressionsToWrite, sourcePlan); ExpressionMapping expressionMappingKeySourceBased = targetExpressionNeedSourceBased - ? targetExpressionMapping.keyPermute(sourceToTargetMapping.inverse()) : targetExpressionMapping; + ? targetExpressionMapping.keyPermute(targetToSourceMapping) : targetExpressionMapping; // target to target replacement expression mapping, because mv is 1:1 so get first element List> flattenExpressionMap = expressionMappingKeySourceBased.flattenMap(); Map targetToTargetReplacementMapping = flattenExpressionMap.get(0); List rewrittenExpressions = new ArrayList<>(); - for (int index = 0; index < sourceShuttledExpressions.size(); index++) { - Expression expressionShuttledToRewrite = sourceShuttledExpressions.get(index); + for (Expression expressionShuttledToRewrite : sourceShuttledExpressions) { if (expressionShuttledToRewrite instanceof Literal) { rewrittenExpressions.add(expressionShuttledToRewrite); continue; @@ -327,39 +386,31 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac // if contains any slot to rewrite, which means can not be rewritten by target, bail out return ImmutableList.of(); } - Expression sourceExpression = sourceExpressionsToWrite.get(index); - if (sourceExpression instanceof NamedExpression - && replacedExpression.nullable() != sourceExpression.nullable()) { - // if enable join eliminate, query maybe inner join and mv maybe outer join. - // If the slot is at null generate side, the nullable maybe different between query and view - // So need to force to consistent. - replacedExpression = sourceExpression.nullable() - ? new Nullable(replacedExpression) : new NonNullable(replacedExpression); - } - if (sourceExpression instanceof NamedExpression) { - NamedExpression sourceNamedExpression = (NamedExpression) sourceExpression; - replacedExpression = new Alias(sourceNamedExpression.getExprId(), replacedExpression, - sourceNamedExpression.getName()); - } rewrittenExpressions.add(replacedExpression); } return rewrittenExpressions; } /** - * Rewrite single expression, the logic is the same with above + * Normalize expression with query, keep the consistency of exprId and nullable props with + * query */ - protected Expression rewriteExpression(Expression sourceExpressionsToWrite, Plan sourcePlan, - ExpressionMapping targetExpressionMapping, SlotMapping sourceToTargetMapping, - boolean targetExpressionNeedSourceBased) { - List expressionToRewrite = new ArrayList<>(); - expressionToRewrite.add(sourceExpressionsToWrite); - List rewrittenExpressions = rewriteExpression(expressionToRewrite, sourcePlan, - targetExpressionMapping, sourceToTargetMapping, targetExpressionNeedSourceBased); - if (rewrittenExpressions.isEmpty()) { - return null; + protected Expression normalizeExpression(Expression sourceExpression, Expression replacedExpression) { + if (sourceExpression instanceof NamedExpression + && replacedExpression.nullable() != sourceExpression.nullable()) { + // if enable join eliminate, query maybe inner join and mv maybe outer join. + // If the slot is at null generate side, the nullable maybe different between query and view + // So need to force to consistent. + replacedExpression = sourceExpression.nullable() + ? new Nullable(replacedExpression) : new NonNullable(replacedExpression); } - return rewrittenExpressions.get(0); + if (sourceExpression instanceof NamedExpression + && !sourceExpression.equals(replacedExpression)) { + NamedExpression sourceNamedExpression = (NamedExpression) sourceExpression; + replacedExpression = new Alias(sourceNamedExpression.getExprId(), replacedExpression, + sourceNamedExpression.getName()); + } + return replacedExpression; } /** @@ -371,7 +422,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac protected SplitPredicate predicatesCompensate( StructInfo queryStructInfo, StructInfo viewStructInfo, - SlotMapping queryToViewSlotMapping, + SlotMapping viewToQuerySlotMapping, ComparisonResult comparisonResult, CascadesContext cascadesContext ) { @@ -379,15 +430,16 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac List queryPulledUpExpressions = ImmutableList.copyOf(comparisonResult.getQueryExpressions()); // set pulled up expression to queryStructInfo predicates and update related predicates if (!queryPulledUpExpressions.isEmpty()) { - queryStructInfo.addPredicates(queryPulledUpExpressions); + queryStructInfo = queryStructInfo.withPredicates( + queryStructInfo.getPredicates().merge(queryPulledUpExpressions)); } List viewPulledUpExpressions = ImmutableList.copyOf(comparisonResult.getViewExpressions()); // set pulled up expression to viewStructInfo predicates and update related predicates if (!viewPulledUpExpressions.isEmpty()) { - viewStructInfo.addPredicates(viewPulledUpExpressions); + viewStructInfo = viewStructInfo.withPredicates( + viewStructInfo.getPredicates().merge(viewPulledUpExpressions)); } // viewEquivalenceClass to query based - SlotMapping viewToQuerySlotMapping = queryToViewSlotMapping.inverse(); // equal predicate compensate final Set equalCompensateConjunctions = Predicates.compensateEquivalence( queryStructInfo, @@ -422,11 +474,11 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac // query has not null reject predicates, so return return SplitPredicate.INVALID_INSTANCE; } + SlotMapping queryToViewMapping = viewToQuerySlotMapping.inverse(); Set queryUsedNeedRejectNullSlotsViewBased = nullRejectPredicates.stream() .map(expression -> TypeUtils.isNotNull(expression).orElse(null)) .filter(Objects::nonNull) - .map(expr -> ExpressionUtils.replace((Expression) expr, - queryToViewSlotMapping.toSlotReferenceMap())) + .map(expr -> ExpressionUtils.replace((Expression) expr, queryToViewMapping.toSlotReferenceMap())) .collect(Collectors.toSet()); if (requireNoNullableViewSlot.stream().anyMatch( set -> Sets.intersection(set, queryUsedNeedRejectNullSlotsViewBased).isEmpty())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java index 9059499d38..8e0f8d6f71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java @@ -28,7 +28,7 @@ import java.util.List; /** * This is responsible for aggregate rewriting according to different pattern - * */ + */ public class MaterializedViewAggregateRule extends AbstractMaterializedViewAggregateRule { public static final MaterializedViewAggregateRule INSTANCE = new MaterializedViewAggregateRule(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java index 93bd1d314b..472e49d3b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; +import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -39,32 +40,30 @@ import java.util.Set; import java.util.stream.Collectors; /** - * This record the predicates which can be pulled up or some other type predicates + * This record the predicates which can be pulled up or some other type predicates. + * Also contains the necessary method for predicates process */ public class Predicates { // Predicates that can be pulled up - private final Set pulledUpPredicates = new HashSet<>(); + private final Set pulledUpPredicates; - private Predicates() { + public Predicates(Set pulledUpPredicates) { + this.pulledUpPredicates = pulledUpPredicates; } - public static Predicates of() { - return new Predicates(); - } - - public static Predicates of(List pulledUpPredicates) { - Predicates predicates = new Predicates(); - pulledUpPredicates.forEach(predicates::addPredicate); - return predicates; + public static Predicates of(Set pulledUpPredicates) { + return new Predicates(pulledUpPredicates); } public Set getPulledUpPredicates() { return pulledUpPredicates; } - public void addPredicate(Expression expression) { - this.pulledUpPredicates.add(expression); + public Predicates merge(Collection predicates) { + Set mergedPredicates = new HashSet<>(predicates); + mergedPredicates.addAll(this.pulledUpPredicates); + return new Predicates(mergedPredicates); } public Expression composedExpression() { @@ -98,8 +97,7 @@ public class Predicates { if (queryEquivalenceClass.isEmpty() && viewEquivalenceClass.isEmpty()) { equalCompensateConjunctions.add(BooleanLiteral.TRUE); } - if (queryEquivalenceClass.isEmpty() - && !viewEquivalenceClass.isEmpty()) { + if (queryEquivalenceClass.isEmpty() && !viewEquivalenceClass.isEmpty()) { return null; } EquivalenceClassSetMapping queryToViewEquivalenceMapping = diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index d79153bdc8..3451d8e7c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -17,7 +17,9 @@ package org.apache.doris.nereids.rules.exploration.mv; +import org.apache.doris.common.Pair; import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge; import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; @@ -44,6 +46,7 @@ import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer; import org.apache.doris.nereids.util.ExpressionUtils; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -60,6 +63,8 @@ import javax.annotation.Nullable; /** * StructInfo for plan, this contains necessary info for query rewrite by materialized view + * the struct info is used by all materialization, so it's struct info should only get, should not + * modify, if wanting to modify, should copy and then modify */ public class StructInfo { public static final JoinPatternChecker JOIN_PATTERN_CHECKER = new JoinPatternChecker(); @@ -70,58 +75,76 @@ public class StructInfo { private static final PredicateCollector PREDICATE_COLLECTOR = new PredicateCollector(); // source data private final Plan originalPlan; - private ObjectId originalPlanId; + private final ObjectId originalPlanId; private final HyperGraph hyperGraph; - private boolean valid = true; + private final boolean valid; // derived data following // top plan which may include project or filter, except for join and scan - private Plan topPlan; + private final Plan topPlan; // bottom plan which top plan only contain join or scan. this is needed by hyper graph - private Plan bottomPlan; - private final List relations = new ArrayList<>(); + private final Plan bottomPlan; + private final List relations; // this is for LogicalCompatibilityContext later - private final Map relationIdStructInfoNodeMap = new HashMap<>(); + private final Map relationIdStructInfoNodeMap; // this recorde the predicates which can pull up, not shuttled private Predicates predicates; // split predicates is shuttled - private SplitPredicate splitPredicate; - private EquivalenceClass equivalenceClass; + private final SplitPredicate splitPredicate; + private final EquivalenceClass equivalenceClass; // Key is the expression shuttled and the value is the origin expression // this is for building LogicalCompatibilityContext later. - private final Map shuttledHashConjunctsToConjunctsMap = new HashMap<>(); + private final Map shuttledHashConjunctsToConjunctsMap; // Record the exprId and the corresponding expr map, this is used by expression shuttled - private final Map namedExprIdAndExprMapping = new HashMap<>(); + private final Map namedExprIdAndExprMapping; - private StructInfo(Plan originalPlan, @Nullable Plan topPlan, @Nullable Plan bottomPlan, HyperGraph hyperGraph) { + /** + * The construct method for StructInfo + */ + public StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperGraph, boolean valid, Plan topPlan, + Plan bottomPlan, List relations, + Map relationIdStructInfoNodeMap, + @Nullable Predicates predicates, + Map shuttledHashConjunctsToConjunctsMap, + Map namedExprIdAndExprMapping) { this.originalPlan = originalPlan; - this.originalPlanId = originalPlan.getGroupExpression() - .map(GroupExpression::getId).orElseGet(() -> new ObjectId(-1)); + this.originalPlanId = originalPlanId; this.hyperGraph = hyperGraph; + this.valid = valid; this.topPlan = topPlan; this.bottomPlan = bottomPlan; - init(); - } - - private void init() { - // split the top plan to two parts by join node - if (topPlan == null || bottomPlan == null) { - PlanSplitContext planSplitContext = new PlanSplitContext(Sets.newHashSet(LogicalJoin.class)); - originalPlan.accept(PLAN_SPLITTER, planSplitContext); - this.bottomPlan = planSplitContext.getBottomPlan(); - this.topPlan = planSplitContext.getTopPlan(); + this.relations = relations; + this.relationIdStructInfoNodeMap = relationIdStructInfoNodeMap; + this.predicates = predicates; + if (predicates == null) { + // collect predicate from top plan which not in hyper graph + Set topPlanPredicates = new HashSet<>(); + topPlan.accept(PREDICATE_COLLECTOR, topPlanPredicates); + this.predicates = Predicates.of(topPlanPredicates); } - collectStructInfoFromGraph(); - initPredicates(); + Pair derivedPredicates = predicatesDerive(this.predicates, originalPlan); + this.splitPredicate = derivedPredicates.key(); + this.equivalenceClass = derivedPredicates.value(); + this.shuttledHashConjunctsToConjunctsMap = shuttledHashConjunctsToConjunctsMap; + this.namedExprIdAndExprMapping = namedExprIdAndExprMapping; } - public void addPredicates(List canPulledUpExpressions) { - canPulledUpExpressions.forEach(this.predicates::addPredicate); - predicatesDerive(); + /** + * Construct StructInfo with new predicates + */ + public StructInfo withPredicates(Predicates predicates) { + return new StructInfo(this.originalPlan, this.originalPlanId, this.hyperGraph, this.valid, this.topPlan, + this.bottomPlan, this.relations, this.relationIdStructInfoNodeMap, predicates, + this.shuttledHashConjunctsToConjunctsMap, this.namedExprIdAndExprMapping); } - private void collectStructInfoFromGraph() { + private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, + Plan topPlan, + Map shuttledHashConjunctsToConjunctsMap, + Map namedExprIdAndExprMapping, + ImmutableList.Builder relationBuilder, + Map relationIdStructInfoNodeMap) { // Collect expression from join condition in hyper graph - this.hyperGraph.getJoinEdges().forEach(edge -> { + for (JoinEdge edge : hyperGraph.getJoinEdges()) { List hashJoinConjuncts = edge.getHashJoinConjuncts(); // shuttle expression in edge for the build of LogicalCompatibilityContext later. // Record the exprId to expr map in the processing to strut info @@ -132,34 +155,31 @@ public class StructInfo { Lists.newArrayList(conjunctExpr), ImmutableSet.of(), ImmutableSet.of()); - this.topPlan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext); + topPlan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext); // Replace expressions by expression map List replacedExpressions = replaceContext.getReplacedExpressions(); shuttledHashConjunctsToConjunctsMap.put(replacedExpressions.get(0), conjunctExpr); // Record this, will be used in top level expression shuttle later, see the method // ExpressionLineageReplacer#visitGroupPlan - this.namedExprIdAndExprMapping.putAll(replaceContext.getExprIdExpressionMap()); + namedExprIdAndExprMapping.putAll(replaceContext.getExprIdExpressionMap()); }); List otherJoinConjuncts = edge.getOtherJoinConjuncts(); if (!otherJoinConjuncts.isEmpty()) { - this.valid = false; + return false; } - }); - if (!this.isValid()) { - return; } // Collect relations from hyper graph which in the bottom plan - this.hyperGraph.getNodes().forEach(node -> { + hyperGraph.getNodes().forEach(node -> { // plan relation collector and set to map Plan nodePlan = node.getPlan(); List nodeRelations = new ArrayList<>(); nodePlan.accept(RELATION_COLLECTOR, nodeRelations); - this.relations.addAll(nodeRelations); + relationBuilder.addAll(nodeRelations); // every node should only have one relation, this is for LogicalCompatibilityContext relationIdStructInfoNodeMap.put(nodeRelations.get(0).getRelationId(), (StructInfoNode) node); }); // Collect expression from where in hyper graph - this.hyperGraph.getFilterEdges().forEach(filterEdge -> { + hyperGraph.getFilterEdges().forEach(filterEdge -> { List filterExpressions = filterEdge.getExpressions(); filterExpressions.forEach(predicate -> { // this is used for LogicalCompatibilityContext @@ -168,28 +188,18 @@ public class StructInfo { ExpressionUtils.shuttleExpressionWithLineage(predicate, topPlan), predicate)); }); }); - } - - private void initPredicates() { - // Collect predicate from top plan which not in hyper graph - this.predicates = Predicates.of(); - Set topPlanPredicates = new HashSet<>(); - topPlan.accept(PREDICATE_COLLECTOR, topPlanPredicates); - topPlanPredicates.forEach(this.predicates::addPredicate); - predicatesDerive(); + return true; } // derive some useful predicate by predicates - private void predicatesDerive() { + private Pair predicatesDerive(Predicates predicates, Plan originalPlan) { // construct equivalenceClass according to equals predicates List shuttledExpression = ExpressionUtils.shuttleExpressionWithLineage( - new ArrayList<>(this.predicates.getPulledUpPredicates()), originalPlan).stream() + new ArrayList<>(predicates.getPulledUpPredicates()), originalPlan).stream() .map(Expression.class::cast) .collect(Collectors.toList()); SplitPredicate splitPredicate = Predicates.splitPredicates(ExpressionUtils.and(shuttledExpression)); - this.splitPredicate = splitPredicate; - - this.equivalenceClass = new EquivalenceClass(); + EquivalenceClass equivalenceClass = new EquivalenceClass(); for (Expression expression : ExpressionUtils.extractConjunction(splitPredicate.getEqualPredicate())) { if (expression instanceof Literal) { continue; @@ -201,6 +211,7 @@ public class StructInfo { (SlotReference) equalTo.getArguments().get(1)); } } + return Pair.of(splitPredicate, equivalenceClass); } /** @@ -216,11 +227,39 @@ public class StructInfo { List structInfos = HyperGraph.toStructInfo(planSplitContext.getBottomPlan()); return structInfos.stream() - .map(hyperGraph -> new StructInfo(originalPlan, planSplitContext.getTopPlan(), + .map(hyperGraph -> StructInfo.of(originalPlan, planSplitContext.getTopPlan(), planSplitContext.getBottomPlan(), hyperGraph)) .collect(Collectors.toList()); } + /** + * The construct method for init StructInfo + */ + public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable Plan bottomPlan, + HyperGraph hyperGraph) { + ObjectId originalPlanId = originalPlan.getGroupExpression() + .map(GroupExpression::getId).orElseGet(() -> new ObjectId(-1)); + // if any of topPlan or bottomPlan is null, split the top plan to two parts by join node + if (topPlan == null || bottomPlan == null) { + PlanSplitContext planSplitContext = new PlanSplitContext(Sets.newHashSet(LogicalJoin.class)); + originalPlan.accept(PLAN_SPLITTER, planSplitContext); + bottomPlan = planSplitContext.getBottomPlan(); + topPlan = planSplitContext.getTopPlan(); + } + // collect struct info fromGraph + ImmutableList.Builder relationBuilder = ImmutableList.builder(); + Map relationIdStructInfoNodeMap = new HashMap<>(); + Map shuttledHashConjunctsToConjunctsMap = new HashMap<>(); + Map namedExprIdAndExprMapping = new HashMap<>(); + boolean valid = collectStructInfoFromGraph(hyperGraph, topPlan, shuttledHashConjunctsToConjunctsMap, + namedExprIdAndExprMapping, + relationBuilder, + relationIdStructInfoNodeMap); + return new StructInfo(originalPlan, originalPlanId, hyperGraph, valid, topPlan, bottomPlan, + relationBuilder.build(), relationIdStructInfoNodeMap, null, shuttledHashConjunctsToConjunctsMap, + namedExprIdAndExprMapping); + } + /** * Build Struct info from group. * Maybe return multi structInfo when original plan already be rewritten by mv diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NormalizeToSlot.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NormalizeToSlot.java index 41f384ac77..1cd56ad129 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NormalizeToSlot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NormalizeToSlot.java @@ -38,10 +38,14 @@ import java.util.function.BiFunction; import java.util.stream.Collectors; import javax.annotation.Nullable; -/** NormalizeToSlot */ +/** + * NormalizeToSlot + */ public interface NormalizeToSlot { - /** NormalizeSlotContext */ + /** + * NormalizeSlotContext + */ class NormalizeToSlotContext { private final Map normalizeToSlotMap; @@ -51,11 +55,11 @@ public interface NormalizeToSlot { /** * build normalization context by follow step. - * 1. collect all exists alias by input parameters existsAliases build a reverted map: expr -> alias - * 2. for all input source expressions, use existsAliasMap to construct triple: - * origin expr, pushed expr and alias to replace origin expr, - * see more detail in {@link NormalizeToSlotTriplet} - * 3. construct a map: original expr -> triple constructed by step 2 + * 1. collect all exists alias by input parameters existsAliases build a reverted map: expr -> alias + * 2. for all input source expressions, use existsAliasMap to construct triple: + * origin expr, pushed expr and alias to replace origin expr, + * see more detail in {@link NormalizeToSlotTriplet} + * 3. construct a map: original expr -> triple constructed by step 2 */ public static NormalizeToSlotContext buildContext( Set existsAliases, Collection sourceExpressions) { @@ -65,7 +69,6 @@ public interface NormalizeToSlot { for (Alias existsAlias : existsAliases) { existsAliasMap.put(existsAlias.child(), existsAlias); } - for (Expression expression : sourceExpressions) { if (normalizeToSlotMap.containsKey(expression)) { continue; @@ -186,7 +189,9 @@ public interface NormalizeToSlot { } } - /** NormalizeToSlotTriplet */ + /** + * NormalizeToSlotTriplet + */ class NormalizeToSlotTriplet { // which expression need to normalized to slot? // e.g. `a + 1` diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java index 00ac71eaf2..3519a983fd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java @@ -199,23 +199,6 @@ public interface TreeNode> { return false; } - /** - * iterate top down and test predicate if any matched. Top-down traverse implicitly. - * @param predicate predicate - * @return the first node which match the predicate - */ - default TreeNode firstMatch(Predicate> predicate) { - if (predicate.test(this)) { - return this; - } - for (NODE_TYPE child : children()) { - if (child.anyMatch(predicate)) { - return child; - } - } - return this; - } - /** * Collect the nodes that satisfied the predicate. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index 5e7ffc1e81..cdacfe95b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -61,6 +61,10 @@ public class LogicalProject extends LogicalUnary projects, CHILD_TYPE child, boolean canEliminate) { + this(projects, ImmutableList.of(), false, canEliminate, ImmutableList.of(child)); + } + public LogicalProject(List projects, List excepts, boolean isDistinct, List child) { this(projects, excepts, isDistinct, true, Optional.empty(), Optional.empty(), child); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index 7a2d9fc27d..fd96ceecb9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -231,7 +231,9 @@ public class ExpressionUtils { Plan plan, Set targetTypes, Set tableIdentifiers) { - + if (expressions.isEmpty()) { + return ImmutableList.of(); + } ExpressionLineageReplacer.ExpressionReplaceContext replaceContext = new ExpressionLineageReplacer.ExpressionReplaceContext( expressions.stream().map(Expression.class::cast).collect(Collectors.toList()), diff --git a/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.out b/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.out index 10c593bd5f..17104fbd73 100644 --- a/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.out +++ b/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.out @@ -255,3 +255,9 @@ -- !query29_0_after -- 8 +-- !query29_1_before -- +0 178.10 1.20 8 + +-- !query29_1_after -- +0 178.10 1.20 8 + diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy index cffb031cb1..4d001af412 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy @@ -171,7 +171,7 @@ suite("aggregate_with_roll_up") { } } - def check_rewrite_with_force_analyze = { mv_sql, query_sql, mv_name -> + def check_rewrite_but_not_chose = { mv_sql, query_sql, mv_name -> sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}""" sql""" @@ -182,16 +182,14 @@ suite("aggregate_with_roll_up") { AS ${mv_sql} """ - sql "analyze table ${mv_name} with sync;" - sql "analyze table lineitem with sync;" - sql "analyze table orders with sync;" - sql "analyze table partsupp with sync;" - def job_name = getJobName(db, mv_name); waitingMTMVTaskFinished(job_name) explain { sql("${query_sql}") - contains("${mv_name}(${mv_name})") + check {result -> + def splitResult = result.split("MaterializedViewRewriteSuccessButNotChose") + splitResult.length == 2 ? splitResult[1].contains(mv_name) : false + } } } @@ -436,19 +434,21 @@ suite("aggregate_with_roll_up") { "o_orderdate, " + "l_partkey, " + "l_suppkey" - def query17_0 = "select t1.l_partkey, t1.l_suppkey, l_shipdate, " + - "sum(o_totalprice), " + - "max(o_totalprice), " + - "min(o_totalprice), " + - "count(*), " + - "count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) " + - "from lineitem t1 " + - "left join orders on t1.l_orderkey = orders.o_orderkey and t1.l_shipdate = o_orderdate " + - "where o_orderdate = '2023-12-11' " + - "group by " + - "l_shipdate, " + - "l_partkey, " + - "l_suppkey" + def query17_0 = """ + select t1.l_partkey, t1.l_suppkey, l_shipdate, + sum(o_totalprice), + max(o_totalprice), + min(o_totalprice), + count(*), + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) + from lineitem t1 + left join orders on t1.l_orderkey = orders.o_orderkey and t1.l_shipdate = o_orderdate + where o_orderdate = '2023-12-11' + group by + l_shipdate, + l_partkey, + l_suppkey; + """ order_qt_query17_0_before "${query17_0}" check_rewrite(mv17_0, query17_0, "mv17_0") order_qt_query17_0_after "${query17_0}" @@ -888,34 +888,38 @@ suite("aggregate_with_roll_up") { // single table // filter + use roll up dimension - def mv1_1 = "select o_orderdate, o_shippriority, o_comment, " + - "sum(o_totalprice) as sum_total, " + - "max(o_totalprice) as max_total, " + - "min(o_totalprice) as min_total, " + - "count(*) as count_all, " + - "bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, " + - "bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 " + - "from orders " + - "group by " + - "o_orderdate, " + - "o_shippriority, " + - "o_comment " - def query1_1 = "select o_shippriority, o_comment, " + - "count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) as cnt_1, " + - "count(distinct case when O_SHIPPRIORITY > 2 and o_orderkey IN (2) then o_custkey else null end) as cnt_2, " + - "sum(o_totalprice), " + - "max(o_totalprice), " + - "min(o_totalprice), " + - "count(*) " + - "from orders " + - "where o_orderdate = '2023-12-09' " + - "group by " + - "o_shippriority, " + - "o_comment " + def mv1_1 = """ + select o_orderdate, o_shippriority, o_comment, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, + bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 + from orders + group by + o_orderdate, + o_shippriority, + o_comment; + """ + def query1_1 = """ + select o_shippriority, o_comment, + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) as cnt_1, + count(distinct case when O_SHIPPRIORITY > 2 and o_orderkey IN (2) then o_custkey else null end) as cnt_2, + sum(o_totalprice), + max(o_totalprice), + min(o_totalprice), + count(*) + from orders + where o_orderdate = '2023-12-09' + group by + o_shippriority, + o_comment; + """ order_qt_query1_1_before "${query1_1}" - // rewrite success, for cbo chose, should force analyze + // rewrite success, but not chose // because data volume is small and mv plan is almost same to query plan - check_rewrite_with_force_analyze(mv1_1, query1_1, "mv1_1") + check_rewrite_but_not_chose(mv1_1, query1_1, "mv1_1") order_qt_query1_1_after "${query1_1}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_1""" @@ -947,9 +951,9 @@ suite("aggregate_with_roll_up") { "o_comment " order_qt_query2_0_before "${query2_0}" - // rewrite success, for cbo chose, should force analyze + // rewrite success, but not chose // because data volume is small and mv plan is almost same to query plan - check_rewrite_with_force_analyze(mv2_0, query2_0, "mv2_0") + check_rewrite_but_not_chose(mv2_0, query2_0, "mv2_0") order_qt_query2_0_after "${query2_0}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_0""" @@ -1078,8 +1082,8 @@ suite("aggregate_with_roll_up") { ifnull(o_totalprice, 0) as price_with_no_null from lineitem left join orders on l_orderkey = o_orderkey and l_shipdate = o_orderdate - ) - select + ) + select count(1) count_all from cte_view_1 cte_view @@ -1090,4 +1094,31 @@ suite("aggregate_with_roll_up") { check_rewrite(mv29_0, query29_0, "mv29_0") order_qt_query29_0_after "${query29_0}" sql """ DROP MATERIALIZED VIEW IF EXISTS mv29_0""" + + // mv and query both are scalar aggregate + def mv29_1 = """ + select + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) cnt_1, + bitmap_union(to_bitmap(case when o_shippriority > 2 and o_orderkey IN (2) then o_custkey else null end)) as cnt_2 + from lineitem + left join orders on l_orderkey = o_orderkey and l_shipdate = o_orderdate; + """ + def query29_1 = """ + select + count(distinct case when O_SHIPPRIORITY > 2 and o_orderkey IN (2) then o_custkey else null end) as cnt_2, + sum(o_totalprice), + min(o_totalprice), + count(*) + from lineitem + left join orders on l_orderkey = o_orderkey and l_shipdate = o_orderdate; + """ + + order_qt_query29_1_before "${query29_1}" + check_rewrite(mv29_1, query29_1, "mv29_1") + order_qt_query29_1_after "${query29_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv29_1""" }