From be81eb1a9bf3b93a8e42dd071cca61641d7d0da5 Mon Sep 17 00:00:00 2001 From: seawinde <149132972+seawinde@users.noreply.github.com> Date: Thu, 7 Dec 2023 20:29:51 +0800 Subject: [PATCH] [feature](nereids) Support inner join query rewrite by materialized view (#27922) Work in process. Support inner join query rewrite by materialized view in some scene. Such as an exmple as following: > mv = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY " + > "from orders " + > "inner join lineitem on lineitem.L_ORDERKEY = orders.O_ORDERKEY " > query = "select lineitem.L_LINENUMBER " + > "from lineitem " + > "inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " --- .../java/org/apache/doris/catalog/MTMV.java | 11 + .../common/MaterializedViewException.java | 45 +++ .../apache/doris/mtmv/MTMVCacheManager.java | 6 + .../java/org/apache/doris/mtmv/MVCache.java | 82 +++++ .../apache/doris/nereids/NereidsPlanner.java | 14 +- .../org/apache/doris/nereids/PlannerHook.java | 38 +++ .../cascades/OptimizeGroupExpressionJob.java | 6 +- .../org/apache/doris/nereids/memo/Group.java | 14 +- .../mv/AbstractMaterializedViewJoinRule.java | 71 ++++- .../mv/AbstractMaterializedViewRule.java | 257 ++++++++++++---- .../exploration/mv/EquivalenceClass.java | 64 +++- .../mv/InitMaterializationContextHook.java | 111 +++++++ .../mv/LogicalCompatibilityContext.java | 100 ++++++ .../mv/MaterializationContext.java | 88 +++++- .../rules/exploration/mv/Predicates.java | 81 +++-- .../exploration/mv/PredicatesSplitter.java | 2 +- .../rules/exploration/mv/StructInfo.java | 288 ++++++++++++++++-- .../mapping/EquivalenceClassSetMapping.java | 68 +++++ .../mv/mapping/ExpressionMapping.java | 100 ++++++ .../exploration/mv/mapping/SlotMapping.java | 36 ++- .../apache/doris/nereids/trees/TreeNode.java | 14 + .../nereids/trees/expressions/ExprId.java | 10 + .../doris/nereids/trees/plans/RelationId.java | 10 + .../trees/plans/commands/ExplainCommand.java | 4 + .../trees/plans/visitor/TableCollector.java | 2 +- .../doris/nereids/util/ExpressionUtils.java | 84 ++++- .../apache/doris/planner/OriginalPlanner.java | 4 + .../org/apache/doris/planner/Planner.java | 3 + .../org/apache/doris/qe/StmtExecutor.java | 4 + .../rules/exploration/mv/MappingTest.java | 2 +- .../expression/PredicatesSplitterTest.java | 7 +- .../doris/nereids/util/PlanChecker.java | 2 +- .../data/nereids_rules_p0/mv/inner_join.out | 9 + .../nereids_rules_p0/mv/inner_join.groovy | 165 ++++++++++ 34 files changed, 1615 insertions(+), 187 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/MaterializedViewException.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/MVCache.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerHook.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/EquivalenceClassSetMapping.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java create mode 100644 regression-test/data/nereids_rules_p0/mv/inner_join.out create mode 100644 regression-test/suites/nereids_rules_p0/mv/inner_join.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index 0c59f56db5..af0691d94f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -30,6 +30,7 @@ import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState; import org.apache.doris.mtmv.MTMVRefreshInfo; import org.apache.doris.mtmv.MTMVRelation; import org.apache.doris.mtmv.MTMVStatus; +import org.apache.doris.mtmv.MVCache; import org.apache.doris.persist.gson.GsonUtils; import com.google.gson.annotations.SerializedName; @@ -61,6 +62,8 @@ public class MTMV extends OlapTable { private Map mvProperties; @SerializedName("r") private MTMVRelation relation; + // Should update after every fresh + private MVCache mvCache; // For deserialization public MTMV() { @@ -116,6 +119,14 @@ public class MTMV extends OlapTable { return relation; } + public MVCache getMvCache() { + return mvCache; + } + + public void setMvCache(MVCache mvCache) { + this.mvCache = mvCache; + } + public MTMVRefreshInfo alterRefreshInfo(MTMVRefreshInfo newRefreshInfo) { return refreshInfo.updateNotNull(newRefreshInfo); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/MaterializedViewException.java b/fe/fe-core/src/main/java/org/apache/doris/common/MaterializedViewException.java new file mode 100644 index 0000000000..27f2adb78b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/MaterializedViewException.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common; + +/** + * MaterializedViewException + */ +public class MaterializedViewException extends UserException { + + public MaterializedViewException(String msg, Throwable cause) { + super(msg, cause); + } + + public MaterializedViewException(Throwable cause) { + super(cause); + } + + public MaterializedViewException(String msg, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(msg, cause, enableSuppression, writableStackTrace); + } + + public MaterializedViewException(String msg) { + super(msg); + } + + public MaterializedViewException(InternalErrorCode errCode, String msg) { + super(errCode, msg); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCacheManager.java index 847f85b907..950fc79ef7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCacheManager.java @@ -48,6 +48,7 @@ import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollecto import org.apache.doris.persist.AlterMTMV; import org.apache.doris.qe.ConnectContext; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.commons.collections.CollectionUtils; @@ -70,6 +71,11 @@ public class MTMVCacheManager implements MTMVHookService { return tableMTMVs.get(table); } + // TODO Implement the method which getting materialized view by tables + public List getAvailableMaterializedView(List tables) { + return ImmutableList.of(); + } + public boolean isAvailableMTMV(MTMV mtmv, ConnectContext ctx) throws AnalysisException, DdlException { // check session variable if enable rewrite if (!ctx.getSessionVariable().isEnableMvRewrite()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MVCache.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MVCache.java new file mode 100644 index 0000000000..b5cf92f87e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MVCache.java @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.catalog.MTMV; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.OriginStatement; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * The cache for materialized view cache + */ +public class MVCache { + + // the materialized view plan which should be optimized by the same rules to query + private final Plan logicalPlan; + // this should be shuttle expression with lineage + private final List mvOutputExpressions; + + public MVCache(MTMV materializedView, Plan logicalPlan, List mvOutputExpressions) { + this.logicalPlan = logicalPlan; + this.mvOutputExpressions = mvOutputExpressions; + } + + public Plan getLogicalPlan() { + return logicalPlan; + } + + public List getMvOutputExpressions() { + return mvOutputExpressions; + } + + public MVCache(Plan logicalPlan, List mvOutputExpressions) { + this.logicalPlan = logicalPlan; + this.mvOutputExpressions = mvOutputExpressions; + } + + public static MVCache from(MTMV mtmv, ConnectContext connectContext) { + LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(mtmv.getQuerySql()); + // TODO: connect context set current db when create mv by use database + StatementContext mvSqlStatementContext = new StatementContext(connectContext, + new OriginStatement(mtmv.getQuerySql(), 0)); + NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext); + + Plan mvRewrittenPlan = + planner.plan(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN); + Plan mvPlan = mvRewrittenPlan instanceof LogicalResultSink + ? (Plan) ((LogicalResultSink) mvRewrittenPlan).child() : mvRewrittenPlan; + // use rewritten plan output expression currently, if expression rewrite fail, + // consider to use the analyzed plan for output expressions only + List mvOutputExpressions = mvRewrittenPlan.getExpressions().stream() + .map(NamedExpression.class::cast) + .collect(Collectors.toList()); + return new MVCache(mvPlan, mvOutputExpressions); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index f579c2c96d..4ac8223cc3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -88,6 +88,7 @@ public class NereidsPlanner extends Planner { private PhysicalPlan physicalPlan; // The cost of optimized plan private double cost = 0; + private List hooks = new ArrayList<>(); public NereidsPlanner(StatementContext statementContext) { this.statementContext = statementContext; @@ -260,15 +261,12 @@ public class NereidsPlanner extends Planner { if (statementContext.getConnectContext().getTables() != null) { cascadesContext.setTables(statementContext.getConnectContext().getTables()); } - if (statementContext.getConnectContext().getSessionVariable().isEnableMaterializedViewRewrite()) { - // TODO Pre handle materialized view to materializationContext and - // call cascadesContext.addMaterializationContext() to add it - } } private void analyze() { LOG.debug("Start analyze plan"); cascadesContext.newAnalyzer().analyze(); + getHooks().forEach(hook -> hook.afterAnalyze(this)); NereidsTracer.logImportantTime("EndAnalyzePlan"); LOG.debug("End analyze plan"); } @@ -525,4 +523,12 @@ public class NereidsPlanner extends Planner { public PhysicalPlan getPhysicalPlan() { return physicalPlan; } + + public List getHooks() { + return hooks; + } + + public void addHook(PlannerHook hook) { + this.hooks.add(hook); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerHook.java new file mode 100644 index 0000000000..18d71b539a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerHook.java @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids; + +/** + * optimize plan process has some phase, such as analyze, rewrite, optimize, generate physical plan + * and so on, this hook give a chance to do something in the planning process. + * For example: after analyze plan when query or explain, we should generate materialization context. + */ +public interface PlannerHook { + + /** + * the hook before analyze + */ + default void beforeAnalyze(NereidsPlanner planner) { + } + + /** + * the hook after analyze + */ + default void afterAnalyze(NereidsPlanner planner) { + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java index 08b7731fd0..72426f0fa1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java @@ -22,7 +22,9 @@ import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.JobType; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.qe.ConnectContext; +import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -46,7 +48,9 @@ public class OptimizeGroupExpressionJob extends Job { countJobExecutionTimesOfGroupExpressions(groupExpression); List implementationRules = getRuleSet().getImplementationRules(); List explorationRules = getExplorationRules(); - if (context.getCascadesContext().getConnectContext().getSessionVariable().isEnableMaterializedViewRewrite()) { + ConnectContext connectContext = context.getCascadesContext().getConnectContext(); + if (connectContext.getSessionVariable().isEnableMaterializedViewRewrite()) { + explorationRules = new ArrayList<>(explorationRules); explorationRules.addAll(getRuleSet().getMaterializedViewRules()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java index 3f20fcc391..2a0e83c72a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java @@ -76,7 +76,7 @@ public class Group { private int chosenGroupExpressionId = -1; - private Optional structInfo = Optional.empty(); + private List structInfos = new ArrayList<>(); /** * Constructor for Group. @@ -541,11 +541,15 @@ public class Group { return TreeStringUtils.treeString(this, toString, getChildren, getExtraPlans, displayExtraPlan); } - public Optional getStructInfo() { - return structInfo; + public List getStructInfos() { + return structInfos; } - public void setStructInfo(StructInfo structInfo) { - this.structInfo = Optional.ofNullable(structInfo); + public void addStructInfo(StructInfo structInfo) { + this.structInfos.add(structInfo); + } + + public void addStructInfo(List structInfos) { + this.structInfos.addAll(structInfos); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java index 899ba216e9..384522c0f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java @@ -17,47 +17,86 @@ package org.apache.doris.nereids.rules.exploration.mv; -import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.Edge; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.AbstractNode; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; +import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.util.ExpressionUtils; +import com.google.common.collect.Sets; + +import java.util.HashSet; import java.util.List; +import java.util.stream.Collectors; /** * AbstractMaterializedViewJoinRule * This is responsible for common join rewriting */ public abstract class AbstractMaterializedViewJoinRule extends AbstractMaterializedViewRule { + private static final HashSet SUPPORTED_JOIN_TYPE_SET = + Sets.newHashSet(JoinType.INNER_JOIN, JoinType.LEFT_OUTER_JOIN); @Override protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInfo, StructInfo viewStructInfo, - RelationMapping queryToViewTableMappings, - Plan tempRewritedPlan) { + SlotMapping queryToViewSlotMappings, + Plan tempRewritedPlan, + MaterializationContext materializationContext) { - // Rewrite top projects, represent the query projects by view - List expressions = rewriteExpression( + List queryShuttleExpression = ExpressionUtils.shuttleExpressionWithLineage( queryStructInfo.getExpressions(), - queryStructInfo, - viewStructInfo, - queryToViewTableMappings, - tempRewritedPlan + queryStructInfo.getOriginalPlan()); + // Rewrite top projects, represent the query projects by view + List expressionsRewritten = rewriteExpression( + queryShuttleExpression, + materializationContext.getViewExpressionIndexMapping(), + queryToViewSlotMappings ); // Can not rewrite, bail out - if (expressions == null) { + if (expressionsRewritten == null + || expressionsRewritten.stream().anyMatch(expr -> !(expr instanceof NamedExpression))) { return null; } - return new LogicalProject<>(expressions, tempRewritedPlan); + // record the group id in materializationContext, and when rewrite again in + // the same group, bail out quickly. + if (queryStructInfo.getOriginalPlan().getGroupExpression().isPresent()) { + materializationContext.addMatchedGroup( + queryStructInfo.getOriginalPlan().getGroupExpression().get().getOwnerGroup().getGroupId()); + } + return new LogicalProject<>( + expressionsRewritten.stream().map(NamedExpression.class::cast).collect(Collectors.toList()), + tempRewritedPlan); } - // Check join is whether valid or not. Support join's input can not contain aggregate - // Only support project, filter, join, logical relation node and - // join condition should be slot reference equals currently + /** + * Check join is whether valid or not. Support join's input can not contain aggregate + * Only support project, filter, join, logical relation node and + * join condition should be slot reference equals currently + */ @Override protected boolean checkPattern(StructInfo structInfo) { - // TODO Should get struct info from hyper graph and check - return false; + HyperGraph hyperGraph = structInfo.getHyperGraph(); + for (AbstractNode node : hyperGraph.getNodes()) { + StructInfoNode structInfoNode = (StructInfoNode) node; + if (!structInfoNode.getPlan().accept(StructInfo.JOIN_PATTERN_CHECKER, + SUPPORTED_JOIN_TYPE_SET)) { + return false; + } + } + for (Edge edge : hyperGraph.getEdges()) { + if (!edge.getJoin().accept(StructInfo.JOIN_PATTERN_CHECKER, + SUPPORTED_JOIN_TYPE_SET)) { + return false; + } + } + return true; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 2dc64ccfea..09b2ea7761 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -19,23 +19,31 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate; -import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionIndexMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.EquivalenceClassSetMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.util.ExpressionUtils; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; /** @@ -53,54 +61,81 @@ public abstract class AbstractMaterializedViewRule { if (materializationContexts.isEmpty()) { return rewriteResults; } - StructInfo queryStructInfo = extractStructInfo(queryPlan, cascadesContext); - // Check query queryPlan + + List queryStructInfos = extractStructInfo(queryPlan, cascadesContext); + // TODO Just Check query queryPlan firstly, support multi later. + StructInfo queryStructInfo = queryStructInfos.get(0); if (!checkPattern(queryStructInfo)) { return rewriteResults; } for (MaterializationContext materializationContext : materializationContexts) { - Plan mvPlan = materializationContext.getMvPlan(); - StructInfo viewStructInfo = extractStructInfo(mvPlan, cascadesContext); + // already rewrite, bail out + if (queryPlan.getGroupExpression().isPresent() + && materializationContext.alreadyRewrite( + queryPlan.getGroupExpression().get().getOwnerGroup().getGroupId())) { + continue; + } + Plan mvPlan = materializationContext.getMtmv().getMvCache().getLogicalPlan(); + List viewStructInfos = extractStructInfo(mvPlan, cascadesContext); + if (viewStructInfos.size() > 1) { + // view struct info should only have one + return rewriteResults; + } + StructInfo viewStructInfo = viewStructInfos.get(0); if (!checkPattern(viewStructInfo)) { continue; } - if (!StructInfo.isGraphLogicalEquals(queryStructInfo.getHyperGraph(), viewStructInfo.getHyperGraph())) { - continue; - } MatchMode matchMode = decideMatchMode(queryStructInfo.getRelations(), viewStructInfo.getRelations()); - if (MatchMode.NOT_MATCH == matchMode) { + if (MatchMode.COMPLETE != matchMode) { continue; } List queryToViewTableMappings = RelationMapping.generate(queryStructInfo.getRelations(), viewStructInfo.getRelations()); + // if any relation in query and view can not map, bail out. + if (queryToViewTableMappings == null) { + return rewriteResults; + } for (RelationMapping queryToViewTableMapping : queryToViewTableMappings) { + SlotMapping queryToViewSlotMapping = SlotMapping.generate(queryToViewTableMapping); + if (queryToViewSlotMapping == null) { + continue; + } + LogicalCompatibilityContext compatibilityContext = + LogicalCompatibilityContext.from(queryToViewTableMapping, queryToViewSlotMapping, + queryStructInfo, viewStructInfo); + // todo outer join compatibility check + if (!StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo, compatibilityContext)) { + continue; + } SplitPredicate compensatePredicates = predicatesCompensate(queryStructInfo, viewStructInfo, - queryToViewTableMapping); + queryToViewSlotMapping); // Can not compensate, bail out - if (compensatePredicates == null || compensatePredicates.isEmpty()) { + if (compensatePredicates.isEmpty()) { continue; } Plan rewritedPlan; - Plan mvScan = materializationContext.getScanPlan(); + Plan mvScan = materializationContext.getMvScanPlan(); if (compensatePredicates.isAlwaysTrue()) { rewritedPlan = mvScan; } else { // Try to rewrite compensate predicates by using mv scan - List rewriteCompensatePredicates = rewriteExpression( + List rewriteCompensatePredicates = rewriteExpression( compensatePredicates.toList(), - queryStructInfo, - viewStructInfo, - queryToViewTableMapping, - mvScan); + materializationContext.getViewExpressionIndexMapping(), + queryToViewSlotMapping); if (rewriteCompensatePredicates.isEmpty()) { continue; } rewritedPlan = new LogicalFilter<>(Sets.newHashSet(rewriteCompensatePredicates), mvScan); } // Rewrite query by view - rewritedPlan = rewriteQueryByView(matchMode, queryStructInfo, viewStructInfo, - queryToViewTableMapping, rewritedPlan); + rewritedPlan = rewriteQueryByView(matchMode, + queryStructInfo, + viewStructInfo, + queryToViewSlotMapping, + rewritedPlan, + materializationContext); if (rewritedPlan == null) { continue; } @@ -110,22 +145,25 @@ public abstract class AbstractMaterializedViewRule { return rewriteResults; } - /**Rewrite query by view, for aggregate or join rewriting should be different inherit class implementation*/ + /** + * Rewrite query by view, for aggregate or join rewriting should be different inherit class implementation + */ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInfo, StructInfo viewStructInfo, - RelationMapping queryToViewTableMappings, - Plan tempRewritedPlan) { + SlotMapping queryToViewSlotMappings, + Plan tempRewritedPlan, + MaterializationContext materializationContext) { return tempRewritedPlan; } - /**Use target output expression to represent the source expression*/ - protected List rewriteExpression(List sourceExpressions, - StructInfo sourceStructInfo, - StructInfo targetStructInfo, - RelationMapping sourceToTargetMapping, - Plan targetScanNode) { - // TODO represent the sourceExpressions by using target scan node + /** + * Use target output expression to represent the source expression + */ + protected List rewriteExpression( + List sourceExpressionsToWrite, + ExpressionMapping mvExprToMvScanExprMapping, + SlotMapping sourceToTargetMapping) { // Firstly, rewrite the target plan output expression using query with inverse mapping // then try to use the mv expression to represent the query. if any of source expressions // can not be represented by mv, return null @@ -138,18 +176,34 @@ public abstract class AbstractMaterializedViewRule { // transform source to: // project(slot 2, 1) // target - List targetTopExpressions = targetStructInfo.getExpressions(); - List shuttledTargetExpressions = ExpressionUtils.shuttleExpressionWithLineage( - targetTopExpressions, targetStructInfo.getOriginalPlan(), Sets.newHashSet(), Sets.newHashSet()); - SlotMapping sourceToTargetSlotMapping = SlotMapping.generate(sourceToTargetMapping); - // mv sql plan expressions transform to query based - List queryBasedExpressions = ExpressionUtils.replace( - shuttledTargetExpressions.stream().map(Expression.class::cast).collect(Collectors.toList()), - sourceToTargetSlotMapping.inverse().getSlotMap()); - // mv sql query based expression and index mapping - ExpressionIndexMapping.generate(queryBasedExpressions); - // TODO visit source expression and replace the expression with expressionIndexMapping - return ImmutableList.of(); + // generate mvSql to mvScan mvExprToMvScanExprMapping, and change mv sql expression to query based + ExpressionMapping mvExprToMvScanExprMappingKeySourceBased = + mvExprToMvScanExprMapping.keyPermute(sourceToTargetMapping.inverse()); + List> flattenExpressionMapping = + mvExprToMvScanExprMappingKeySourceBased.flattenMap(); + // view to view scan expression is 1:1 so get first element + Map mvSqlToMvScanMappingQueryBased = + flattenExpressionMapping.get(0); + + List rewrittenExpressions = new ArrayList<>(); + for (Expression expressionToRewrite : sourceExpressionsToWrite) { + if (expressionToRewrite instanceof Literal) { + rewrittenExpressions.add(expressionToRewrite); + continue; + } + final Set slotsToRewrite = + expressionToRewrite.collectToSet(expression -> expression instanceof Slot); + boolean wiAlias = expressionToRewrite instanceof NamedExpression; + Expression replacedExpression = ExpressionUtils.replace(expressionToRewrite, + mvSqlToMvScanMappingQueryBased, + wiAlias); + if (replacedExpression.anyMatch(slotsToRewrite::contains)) { + // if contains any slot to rewrite, which means can not be rewritten by target, bail out + return null; + } + rewrittenExpressions.add(replacedExpression); + } + return rewrittenExpressions; } /** @@ -161,21 +215,109 @@ public abstract class AbstractMaterializedViewRule { protected SplitPredicate predicatesCompensate( StructInfo queryStructInfo, StructInfo viewStructInfo, - RelationMapping queryToViewTableMapping + SlotMapping queryToViewSlotMapping ) { - // TODO Equal predicate compensate EquivalenceClass queryEquivalenceClass = queryStructInfo.getEquivalenceClass(); EquivalenceClass viewEquivalenceClass = viewStructInfo.getEquivalenceClass(); + // viewEquivalenceClass to query based + Map viewToQuerySlotMapping = queryToViewSlotMapping.inverse() + .toSlotReferenceMap(); + EquivalenceClass viewEquivalenceClassQueryBased = viewEquivalenceClass.permute(viewToQuerySlotMapping); + if (viewEquivalenceClassQueryBased == null) { + return SplitPredicate.empty(); + } + final List equalCompensateConjunctions = new ArrayList<>(); + if (queryEquivalenceClass.isEmpty() && viewEquivalenceClass.isEmpty()) { + equalCompensateConjunctions.add(BooleanLiteral.of(true)); + } if (queryEquivalenceClass.isEmpty() && !viewEquivalenceClass.isEmpty()) { - return null; + return SplitPredicate.empty(); } - // TODO range predicates and residual predicates compensate - return SplitPredicate.empty(); + EquivalenceClassSetMapping queryToViewEquivalenceMapping = + EquivalenceClassSetMapping.generate(queryEquivalenceClass, viewEquivalenceClassQueryBased); + // can not map all target equivalence class, can not compensate + if (queryToViewEquivalenceMapping.getEquivalenceClassSetMap().size() + < viewEquivalenceClass.getEquivalenceSetList().size()) { + return SplitPredicate.empty(); + } + // do equal compensate + Set> mappedQueryEquivalenceSet = + queryToViewEquivalenceMapping.getEquivalenceClassSetMap().keySet(); + queryEquivalenceClass.getEquivalenceSetList().forEach( + queryEquivalenceSet -> { + // compensate the equivalence in query but not in view + if (!mappedQueryEquivalenceSet.contains(queryEquivalenceSet)) { + Iterator iterator = queryEquivalenceSet.iterator(); + SlotReference first = iterator.next(); + while (iterator.hasNext()) { + Expression equals = new EqualTo(first, iterator.next()); + equalCompensateConjunctions.add(equals); + } + } else { + // compensate the equivalence both in query and view, but query has more equivalence + Set viewEquivalenceSet = + queryToViewEquivalenceMapping.getEquivalenceClassSetMap().get(queryEquivalenceSet); + Set copiedQueryEquivalenceSet = new HashSet<>(queryEquivalenceSet); + copiedQueryEquivalenceSet.removeAll(viewEquivalenceSet); + SlotReference first = viewEquivalenceSet.iterator().next(); + for (SlotReference slotReference : copiedQueryEquivalenceSet) { + Expression equals = new EqualTo(first, slotReference); + equalCompensateConjunctions.add(equals); + } + } + } + ); + // TODO range predicates and residual predicates compensate, Simplify implementation. + SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate(); + SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate(); + + // range compensate + List rangeCompensate = new ArrayList<>(); + Expression queryRangePredicate = querySplitPredicate.getRangePredicate(); + Expression viewRangePredicate = viewSplitPredicate.getRangePredicate(); + Expression viewRangePredicateQueryBased = + ExpressionUtils.replace(viewRangePredicate, viewToQuerySlotMapping); + + Set queryRangeSet = + Sets.newHashSet(ExpressionUtils.extractConjunction(queryRangePredicate)); + Set viewRangeQueryBasedSet = + Sets.newHashSet(ExpressionUtils.extractConjunction(viewRangePredicateQueryBased)); + // query range predicate can not contain all view range predicate when view have range predicate, bail out + if (!viewRangePredicateQueryBased.equals(BooleanLiteral.TRUE) + && !queryRangeSet.containsAll(viewRangeQueryBasedSet)) { + return SplitPredicate.empty(); + } + queryRangeSet.removeAll(viewRangeQueryBasedSet); + rangeCompensate.addAll(queryRangeSet); + + // residual compensate + List residualCompensate = new ArrayList<>(); + Expression queryResidualPredicate = querySplitPredicate.getResidualPredicate(); + Expression viewResidualPredicate = viewSplitPredicate.getResidualPredicate(); + Expression viewResidualPredicateQueryBased = + ExpressionUtils.replace(viewResidualPredicate, viewToQuerySlotMapping); + Set queryResidualSet = + Sets.newHashSet(ExpressionUtils.extractConjunction(queryResidualPredicate)); + Set viewResidualQueryBasedSet = + Sets.newHashSet(ExpressionUtils.extractConjunction(viewResidualPredicateQueryBased)); + // query residual predicate can not contain all view residual predicate when view have residual predicate, + // bail out + if (!viewResidualPredicateQueryBased.equals(BooleanLiteral.TRUE) + && !queryResidualSet.containsAll(viewResidualQueryBasedSet)) { + return SplitPredicate.empty(); + } + queryResidualSet.removeAll(viewResidualQueryBasedSet); + residualCompensate.addAll(queryResidualSet); + + return SplitPredicate.of(ExpressionUtils.and(equalCompensateConjunctions), + rangeCompensate.isEmpty() ? BooleanLiteral.of(true) : ExpressionUtils.and(rangeCompensate), + residualCompensate.isEmpty() ? BooleanLiteral.of(true) : ExpressionUtils.and(residualCompensate)); } /** * Decide the match mode + * * @see MatchMode */ private MatchMode decideMatchMode(List queryRelations, List viewRelations) { @@ -205,20 +347,17 @@ public abstract class AbstractMaterializedViewRule { /** * Extract struct info from plan, support to get struct info from logical plan or plan in group. */ - protected StructInfo extractStructInfo(Plan plan, CascadesContext cascadesContext) { - + protected List extractStructInfo(Plan plan, CascadesContext cascadesContext) { if (plan.getGroupExpression().isPresent() - && plan.getGroupExpression().get().getOwnerGroup().getStructInfo().isPresent()) { - Group belongGroup = plan.getGroupExpression().get().getOwnerGroup(); - return belongGroup.getStructInfo().get(); + && !plan.getGroupExpression().get().getOwnerGroup().getStructInfos().isEmpty()) { + return plan.getGroupExpression().get().getOwnerGroup().getStructInfos(); } else { - // TODO build graph from plan and extract struct from graph and set to group if exist - // Should get structInfo from hyper graph and add into current group - StructInfo structInfo = StructInfo.of(plan); + // build struct info and add them to current group + List structInfos = StructInfo.of(plan); if (plan.getGroupExpression().isPresent()) { - plan.getGroupExpression().get().getOwnerGroup().setStructInfo(structInfo); + plan.getGroupExpression().get().getOwnerGroup().addStructInfo(structInfos); } - return structInfo; + return structInfos; } } @@ -229,7 +368,7 @@ public abstract class AbstractMaterializedViewRule { if (structInfo.getRelations().isEmpty()) { return false; } - return false; + return true; } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java index d140582aa6..23249cf5ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java @@ -20,6 +20,8 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.nereids.trees.expressions.SlotReference; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -31,11 +33,24 @@ import java.util.Set; */ public class EquivalenceClass { - private final Map> equivalenceSlotMap = new LinkedHashMap<>(); + /** + * eg: column a = b + * this will be + * { + * a: [a, b], + * b: [a, b] + * } + */ + private Map> equivalenceSlotMap = new LinkedHashMap<>(); + private List> equivalenceSlotList; public EquivalenceClass() { } + public EquivalenceClass(Map> equivalenceSlotMap) { + this.equivalenceSlotMap = equivalenceSlotMap; + } + /** * EquivalenceClass */ @@ -82,11 +97,48 @@ public class EquivalenceClass { } /** - * EquivalenceClass + * EquivalenceClass permute */ - public List> getEquivalenceValues() { - List> values = new ArrayList<>(); - equivalenceSlotMap.values().forEach(each -> values.add(each)); - return values; + public EquivalenceClass permute(Map mapping) { + + Map> permutedEquivalenceSlotMap = new HashMap<>(); + for (Map.Entry> slotReferenceSetEntry : equivalenceSlotMap.entrySet()) { + SlotReference mappedSlotReferenceKey = mapping.get(slotReferenceSetEntry.getKey()); + if (mappedSlotReferenceKey == null) { + // can not permute then need to return null + return null; + } + Set equivalenceValueSet = slotReferenceSetEntry.getValue(); + final Set mappedSlotReferenceSet = new HashSet<>(); + for (SlotReference target : equivalenceValueSet) { + SlotReference mappedSlotReferenceValue = mapping.get(target); + if (mappedSlotReferenceValue == null) { + return null; + } + mappedSlotReferenceSet.add(mappedSlotReferenceValue); + } + permutedEquivalenceSlotMap.put(mappedSlotReferenceKey, mappedSlotReferenceSet); + } + return new EquivalenceClass(permutedEquivalenceSlotMap); + } + + /** + * Return the list of equivalence set, remove duplicate + */ + public List> getEquivalenceSetList() { + + if (equivalenceSlotList != null) { + return equivalenceSlotList; + } + List> equivalenceSets = new ArrayList<>(); + Set> visited = new HashSet<>(); + equivalenceSlotMap.values().forEach(slotSet -> { + if (!visited.contains(slotSet)) { + equivalenceSets.add(slotSet); + } + visited.add(slotSet); + }); + this.equivalenceSlotList = equivalenceSets; + return this.equivalenceSlotList; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java new file mode 100644 index 0000000000..24dad225e9 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv; + +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.mtmv.MTMVCacheManager; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.PlannerHook; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.visitor.TableCollector; +import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** If enable query rewrite with mv, should init materialization context after analyze*/ +public class InitMaterializationContextHook implements PlannerHook { + + public static final Logger LOG = LogManager.getLogger(InitMaterializationContextHook.class); + public static final InitMaterializationContextHook INSTANCE = new InitMaterializationContextHook(); + + @Override + public void afterAnalyze(NereidsPlanner planner) { + initMaterializationContext(planner.getCascadesContext()); + } + + private void initMaterializationContext(CascadesContext cascadesContext) { + + Plan rewritePlan = cascadesContext.getRewritePlan(); + TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet()); + rewritePlan.accept(TableCollector.INSTANCE, collectorContext); + List collectedTables = collectorContext.getCollectedTables(); + if (collectedTables.isEmpty()) { + return; + } + List baseTableUsed = + collectedTables.stream().map(BaseTableInfo::new).collect(Collectors.toList()); + // TODO the logic should be move to MTMVCacheManager later when getAvailableMaterializedView is ready in + // MV Cache manager + Env env = cascadesContext.getConnectContext().getEnv(); + MTMVCacheManager cacheManager = env.getMtmvService().getCacheManager(); + Set materializedViews = new HashSet<>(); + for (BaseTableInfo baseTableInfo : baseTableUsed) { + Set mtmvsByBaseTable = cacheManager.getMtmvsByBaseTable(baseTableInfo); + if (mtmvsByBaseTable == null || mtmvsByBaseTable.isEmpty()) { + continue; + } + materializedViews.addAll(mtmvsByBaseTable); + } + if (materializedViews.isEmpty()) { + return; + } + materializedViews.forEach(mvBaseTableInfo -> { + try { + MTMV materializedView = (MTMV) Env.getCurrentInternalCatalog() + .getDbOrMetaException(mvBaseTableInfo.getDbId()) + .getTableOrMetaException(mvBaseTableInfo.getTableId(), TableType.MATERIALIZED_VIEW); + + String qualifiedName = materializedView.getQualifiedName(); + // generate outside, maybe add partition filter in the future + Plan mvScan = new LogicalOlapScan(cascadesContext.getStatementContext().getNextRelationId(), + (OlapTable) materializedView, + ImmutableList.of(qualifiedName), + Lists.newArrayList(materializedView.getId()), + Lists.newArrayList(), + Optional.empty()); + List mvProjects = mvScan.getOutput().stream().map(NamedExpression.class::cast) + .collect(Collectors.toList()); + mvScan = new LogicalProject(mvProjects, mvScan); + cascadesContext.addMaterializationContext( + MaterializationContext.fromMaterializedView(materializedView, mvScan, cascadesContext)); + } catch (MetaNotFoundException metaNotFoundException) { + LOG.error(mvBaseTableInfo.toString() + " can not find corresponding materialized view."); + } + }); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java new file mode 100644 index 0000000000..3a7cd1d2bc --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv; + +import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; +import org.apache.doris.nereids.rules.exploration.mv.mapping.Mapping.MappedRelation; +import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.util.ExpressionUtils; + +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; + +import java.util.HashMap; +import java.util.Map; + +/** + * For outer join we should check the outer join compatibility between query and view + */ +public class LogicalCompatibilityContext { + private BiMap queryToViewNodeMapping; + private BiMap queryToViewEdgeExpressionMapping; + + public LogicalCompatibilityContext(BiMap queryToViewNodeMapping, + BiMap queryToViewEdgeExpressionMapping) { + this.queryToViewNodeMapping = queryToViewNodeMapping; + this.queryToViewEdgeExpressionMapping = queryToViewEdgeExpressionMapping; + } + + public BiMap getQueryToViewNodeMapping() { + return queryToViewNodeMapping; + } + + public BiMap getQueryToViewEdgeExpressionMapping() { + return queryToViewEdgeExpressionMapping; + } + + /** + * generate logical compatibility context + */ + public static LogicalCompatibilityContext from(RelationMapping relationMapping, + SlotMapping slotMapping, + StructInfo queryStructInfo, + StructInfo viewStructInfo) { + // init node mapping + BiMap queryToViewNodeMapping = HashBiMap.create(); + Map queryRelationIdStructInfoNodeMap + = queryStructInfo.getRelationIdStructInfoNodeMap(); + Map viewRelationIdStructInfoNodeMap + = viewStructInfo.getRelationIdStructInfoNodeMap(); + for (Map.Entry relationMappingEntry : + relationMapping.getMappedRelationMap().entrySet()) { + StructInfoNode queryStructInfoNode = queryRelationIdStructInfoNodeMap.get( + relationMappingEntry.getKey().getRelationId()); + StructInfoNode viewStructInfoNode = viewRelationIdStructInfoNodeMap.get( + relationMappingEntry.getValue().getRelationId()); + if (queryStructInfoNode != null && viewStructInfoNode != null) { + queryToViewNodeMapping.put(queryStructInfoNode, viewStructInfoNode); + } + } + // init expression mapping + Map viewToQuerySlotMapping = slotMapping.inverse().toSlotReferenceMap(); + Map queryShuttledExprToExprMap = + queryStructInfo.getShuttledHashConjunctsToConjunctsMap(); + Map viewShuttledExprToExprMap = + viewStructInfo.getShuttledHashConjunctsToConjunctsMap(); + final Map viewEdgeToConjunctsMapQueryBased = new HashMap<>(); + viewShuttledExprToExprMap.forEach((shuttledExpr, expr) -> { + viewEdgeToConjunctsMapQueryBased.put( + ExpressionUtils.replace(shuttledExpr, viewToQuerySlotMapping), + expr); + }); + BiMap queryToViewEdgeMapping = HashBiMap.create(); + queryShuttledExprToExprMap.forEach((exprSet, edge) -> { + Expression viewExpr = viewEdgeToConjunctsMapQueryBased.get(exprSet); + if (viewExpr != null) { + queryToViewEdgeMapping.put(edge, viewExpr); + } + }); + return new LogicalCompatibilityContext(queryToViewNodeMapping, queryToViewEdgeMapping); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index b0de1ccfa4..336c627da6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -17,52 +17,110 @@ package org.apache.doris.nereids.rules.exploration.mv; +import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.Table; -import org.apache.doris.catalog.View; +import org.apache.doris.mtmv.MVCache; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.memo.GroupId; +import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.util.ExpressionUtils; + +import com.google.common.collect.ImmutableList; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; /** * Maintain the context for query rewrite by materialized view */ public class MaterializationContext { - // TODO add MaterializedView class - private final Plan mvPlan; - private final CascadesContext context; + private MTMV mtmv; + // Should use stmt id generator in query context + private final Plan mvScanPlan; private final List baseTables; - private final List baseViews; + private final List
baseViews; // Group ids that are rewritten by this mv to reduce rewrite times private final Set matchedGroups = new HashSet<>(); - private final Plan scanPlan; + // generate form mv scan plan + private ExpressionMapping viewExpressionMapping; - public MaterializationContext(Plan mvPlan, CascadesContext context, - List
baseTables, List baseViews, Plan scanPlan) { - this.mvPlan = mvPlan; - this.context = context; + /** + * MaterializationContext, this contains necessary info for query rewriting by mv + */ + public MaterializationContext(MTMV mtmv, + Plan mvScanPlan, + CascadesContext cascadesContext, + List
baseTables, + List
baseViews) { + this.mtmv = mtmv; + this.mvScanPlan = mvScanPlan; this.baseTables = baseTables; this.baseViews = baseViews; - this.scanPlan = scanPlan; + MVCache mvCache = mtmv.getMvCache(); + // TODO This logic should move to materialized view cache manager + if (mvCache == null) { + mvCache = MVCache.from(mtmv, cascadesContext.getConnectContext()); + mtmv.setMvCache(mvCache); + } + List mvOutputExpressions = mvCache.getMvOutputExpressions(); + // mv output expression shuttle, this will be used to expression rewrite + mvOutputExpressions = + ExpressionUtils.shuttleExpressionWithLineage(mvOutputExpressions, mvCache.getLogicalPlan()).stream() + .map(NamedExpression.class::cast) + .collect(Collectors.toList()); + this.viewExpressionMapping = ExpressionMapping.generate( + mvOutputExpressions, + mvScanPlan.getExpressions()); } public Set getMatchedGroups() { return matchedGroups; } + public boolean alreadyRewrite(GroupId groupId) { + return this.matchedGroups.contains(groupId); + } + public void addMatchedGroup(GroupId groupId) { matchedGroups.add(groupId); } - public Plan getMvPlan() { - return mvPlan; + public MTMV getMtmv() { + return mtmv; } - public Plan getScanPlan() { - return scanPlan; + public Plan getMvScanPlan() { + return mvScanPlan; + } + + public List
getBaseTables() { + return baseTables; + } + + public List
getBaseViews() { + return baseViews; + } + + public ExpressionMapping getViewExpressionIndexMapping() { + return viewExpressionMapping; + } + + /** + * MaterializationContext fromMaterializedView + */ + public static MaterializationContext fromMaterializedView(MTMV materializedView, + Plan mvScanPlan, + CascadesContext cascadesContext) { + return new MaterializationContext( + materializedView, + mvScanPlan, + cascadesContext, + ImmutableList.of(), + ImmutableList.of()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java index 43aab2389c..96f89a513c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java @@ -23,36 +23,48 @@ import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; +import java.util.ArrayList; import java.util.List; -import java.util.Set; +import java.util.Optional; +import java.util.stream.Collectors; /** * This record the predicates which can be pulled up or some other type predicates - * */ + */ public class Predicates { // Predicates that can be pulled up - private final Set pulledUpPredicates; + private final List pulledUpPredicates = new ArrayList<>(); - public Predicates(Set pulledUpPredicates) { - this.pulledUpPredicates = pulledUpPredicates; + private Predicates() { } - public static Predicates of(Set pulledUpPredicates) { - return new Predicates(pulledUpPredicates); + public static Predicates of() { + return new Predicates(); } - public Set getPulledUpPredicates() { + public static Predicates of(List pulledUpPredicates) { + Predicates predicates = new Predicates(); + pulledUpPredicates.forEach(predicates::addPredicate); + return predicates; + } + + public List getPulledUpPredicates() { return pulledUpPredicates; } + public void addPredicate(Expression expression) { + this.pulledUpPredicates.add(expression); + } + public Expression composedExpression() { - return ExpressionUtils.and(pulledUpPredicates); + return ExpressionUtils.and(pulledUpPredicates.stream().map(Expression.class::cast) + .collect(Collectors.toList())); } /** * Split the expression to equal, range and residual predicate. - * */ + */ public static SplitPredicate splitPredicates(Expression expression) { PredicatesSplitter predicatesSplit = new PredicatesSplitter(expression); return predicatesSplit.getSplitPredicate(); @@ -60,28 +72,28 @@ public class Predicates { /** * The split different representation for predicate expression, such as equal, range and residual predicate. - * */ + */ public static final class SplitPredicate { - private final Expression equalPredicate; - private final Expression rangePredicate; - private final Expression residualPredicate; + private Optional equalPredicate; + private Optional rangePredicate; + private Optional residualPredicate; public SplitPredicate(Expression equalPredicate, Expression rangePredicate, Expression residualPredicate) { - this.equalPredicate = equalPredicate; - this.rangePredicate = rangePredicate; - this.residualPredicate = residualPredicate; + this.equalPredicate = Optional.ofNullable(equalPredicate); + this.rangePredicate = Optional.ofNullable(rangePredicate); + this.residualPredicate = Optional.ofNullable(residualPredicate); } public Expression getEqualPredicate() { - return equalPredicate; + return equalPredicate.orElse(BooleanLiteral.TRUE); } public Expression getRangePredicate() { - return rangePredicate; + return rangePredicate.orElse(BooleanLiteral.TRUE); } public Expression getResidualPredicate() { - return residualPredicate; + return residualPredicate.orElse(BooleanLiteral.TRUE); } public static SplitPredicate empty() { @@ -90,7 +102,7 @@ public class Predicates { /** * SplitPredicate construct - * */ + */ public static SplitPredicate of(Expression equalPredicates, Expression rangePredicates, Expression residualPredicates) { @@ -99,27 +111,32 @@ public class Predicates { /** * isEmpty - * */ + */ public boolean isEmpty() { - return equalPredicate == null - && rangePredicate == null - && residualPredicate == null; + return !equalPredicate.isPresent() + && !rangePredicate.isPresent() + && !residualPredicate.isPresent(); } public List toList() { - return ImmutableList.of(equalPredicate, rangePredicate, residualPredicate); + return ImmutableList.of(equalPredicate.orElse(BooleanLiteral.TRUE), + rangePredicate.orElse(BooleanLiteral.TRUE), + residualPredicate.orElse(BooleanLiteral.TRUE)); } /** * Check the predicates in SplitPredicate is whether all true or not */ public boolean isAlwaysTrue() { - return equalPredicate instanceof BooleanLiteral - && rangePredicate instanceof BooleanLiteral - && residualPredicate instanceof BooleanLiteral - && ((BooleanLiteral) equalPredicate).getValue() - && ((BooleanLiteral) rangePredicate).getValue() - && ((BooleanLiteral) residualPredicate).getValue(); + Expression equalExpr = equalPredicate.orElse(BooleanLiteral.TRUE); + Expression rangeExpr = rangePredicate.orElse(BooleanLiteral.TRUE); + Expression residualExpr = residualPredicate.orElse(BooleanLiteral.TRUE); + return equalExpr instanceof BooleanLiteral + && rangeExpr instanceof BooleanLiteral + && residualExpr instanceof BooleanLiteral + && ((BooleanLiteral) equalExpr).getValue() + && ((BooleanLiteral) rangeExpr).getValue() + && ((BooleanLiteral) residualExpr).getValue(); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java index e90d6583ea..5f9a5e242d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java @@ -86,7 +86,7 @@ public class PredicatesSplitter { residualPredicates.add(compoundPredicate); return null; } - return super.visit(compoundPredicate, context); + return super.visitCompoundPredicate(compoundPredicate, context); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index ff9dee3ca6..5be9eee897 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -18,56 +18,169 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph; +import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import org.apache.doris.nereids.trees.plans.algebra.Filter; +import org.apache.doris.nereids.trees.plans.algebra.Join; +import org.apache.doris.nereids.trees.plans.algebra.Project; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import org.apache.doris.nereids.util.ExpressionUtils; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Nullable; /** * StructInfo */ public class StructInfo { - private final List relations; - private final Predicates predicates; - // Used by predicate compensation - private final EquivalenceClass equivalenceClass; + public static final JoinPatternChecker JOIN_PATTERN_CHECKER = new JoinPatternChecker(); + // struct info splitter + public static final PlanSplitter PLAN_SPLITTER = new PlanSplitter(); + private static final RelationCollector RELATION_COLLECTOR = new RelationCollector(); + private static final PredicateCollector PREDICATE_COLLECTOR = new PredicateCollector(); + // source data private final Plan originalPlan; private final HyperGraph hyperGraph; + private boolean valid = true; + // derived data following + // top plan which may include project or filter, except for join and scan + private Plan topPlan; + // bottom plan which top plan only contain join or scan. this is needed by hyper graph + private Plan bottomPlan; + private final List relations = new ArrayList<>(); + // this is for LogicalCompatibilityContext later + private final Map relationIdStructInfoNodeMap = new HashMap<>(); + private Predicates predicates; + private SplitPredicate splitPredicate; + private EquivalenceClass equivalenceClass; + // this is for LogicalCompatibilityContext later + private final Map shuttledHashConjunctsToConjunctsMap = new HashMap<>(); - private StructInfo(List relations, - Predicates predicates, - Plan originalPlan, - HyperGraph hyperGraph) { - this.relations = relations; - this.predicates = predicates; + private StructInfo(Plan originalPlan, @Nullable Plan topPlan, @Nullable Plan bottomPlan, HyperGraph hyperGraph) { this.originalPlan = originalPlan; this.hyperGraph = hyperGraph; + this.topPlan = topPlan; + this.bottomPlan = bottomPlan; + init(); + } + + private void init() { + + if (topPlan == null || bottomPlan == null) { + PlanSplitContext planSplitContext = new PlanSplitContext(Sets.newHashSet(LogicalJoin.class)); + originalPlan.accept(PLAN_SPLITTER, planSplitContext); + this.bottomPlan = planSplitContext.getBottomPlan(); + this.topPlan = planSplitContext.getTopPlan(); + } + + this.predicates = Predicates.of(); + // Collect predicate from join condition in hyper graph + this.hyperGraph.getEdges().forEach(edge -> { + List hashJoinConjuncts = edge.getHashJoinConjuncts(); + hashJoinConjuncts.forEach(conjunctExpr -> { + predicates.addPredicate(conjunctExpr); + // shuttle expression in edge for LogicalCompatibilityContext later + shuttledHashConjunctsToConjunctsMap.put( + ExpressionUtils.shuttleExpressionWithLineage( + Lists.newArrayList(conjunctExpr), edge.getJoin()).get(0), + conjunctExpr); + }); + List otherJoinConjuncts = edge.getOtherJoinConjuncts(); + if (!otherJoinConjuncts.isEmpty()) { + this.valid = false; + } + }); + if (!this.isValid()) { + return; + } + + // Collect predicate from filter node in hyper graph + this.hyperGraph.getNodes().forEach(node -> { + // plan relation collector and set to map + Plan nodePlan = node.getPlan(); + List nodeRelations = new ArrayList<>(); + nodePlan.accept(RELATION_COLLECTOR, nodeRelations); + this.relations.addAll(nodeRelations); + // every node should only have one relation, this is for LogicalCompatibilityContext + relationIdStructInfoNodeMap.put(nodeRelations.get(0).getRelationId(), (StructInfoNode) node); + + // if inner join add where condition + Set predicates = new HashSet<>(); + nodePlan.accept(PREDICATE_COLLECTOR, predicates); + predicates.forEach(this.predicates::addPredicate); + }); + + // TODO Collect predicate from top plan not in hyper graph, should optimize, twice now + Set topPlanPredicates = new HashSet<>(); + topPlan.accept(PREDICATE_COLLECTOR, topPlanPredicates); + topPlanPredicates.forEach(this.predicates::addPredicate); + // construct equivalenceClass according to equals predicates this.equivalenceClass = new EquivalenceClass(); - SplitPredicate splitPredicate = Predicates.splitPredicates(predicates.composedExpression()); + List shuttledExpression = ExpressionUtils.shuttleExpressionWithLineage( + this.predicates.getPulledUpPredicates(), originalPlan).stream() + .map(Expression.class::cast) + .collect(Collectors.toList()); + SplitPredicate splitPredicate = Predicates.splitPredicates(ExpressionUtils.and(shuttledExpression)); + this.splitPredicate = splitPredicate; for (Expression expression : ExpressionUtils.extractConjunction(splitPredicate.getEqualPredicate())) { - EqualTo equalTo = (EqualTo) expression; - equivalenceClass.addEquivalenceClass( - (SlotReference) equalTo.getArguments().get(0), - (SlotReference) equalTo.getArguments().get(1)); + if (expression instanceof BooleanLiteral && ((BooleanLiteral) expression).getValue()) { + continue; + } + if (expression instanceof EqualTo) { + EqualTo equalTo = (EqualTo) expression; + equivalenceClass.addEquivalenceClass( + (SlotReference) equalTo.getArguments().get(0), + (SlotReference) equalTo.getArguments().get(1)); + } } } - public static StructInfo of(Plan originalPlan) { - // TODO build graph from original plan and get relations and predicates from graph - return new StructInfo(null, null, originalPlan, null); + /** + * Build Struct info from plan. + * Maybe return multi structInfo when original plan already be rewritten by mv + */ + public static List of(Plan originalPlan) { + // TODO only consider the inner join currently, Should support outer join + // Split plan by the boundary which contains multi child + PlanSplitContext planSplitContext = new PlanSplitContext(Sets.newHashSet(LogicalJoin.class)); + originalPlan.accept(PLAN_SPLITTER, planSplitContext); + + List structInfos = HyperGraph.toStructInfo(planSplitContext.getBottomPlan()); + return structInfos.stream() + .map(hyperGraph -> new StructInfo(originalPlan, planSplitContext.getTopPlan(), + planSplitContext.getBottomPlan(), hyperGraph)) + .collect(Collectors.toList()); } + /** + * Build Struct info from group. + * Maybe return multi structInfo when original plan already be rewritten by mv + */ public static StructInfo of(Group group) { // TODO build graph from original plan and get relations and predicates from graph - return new StructInfo(null, null, group.getLogicalExpression().getPlan(), null); + return null; } public List getRelations() { @@ -90,6 +203,30 @@ public class StructInfo { return hyperGraph; } + public SplitPredicate getSplitPredicate() { + return splitPredicate; + } + + public boolean isValid() { + return valid; + } + + public Plan getTopPlan() { + return topPlan; + } + + public Plan getBottomPlan() { + return bottomPlan; + } + + public Map getRelationIdStructInfoNodeMap() { + return relationIdStructInfoNodeMap; + } + + public Map getShuttledHashConjunctsToConjunctsMap() { + return shuttledHashConjunctsToConjunctsMap; + } + public List getExpressions() { return originalPlan instanceof LogicalProject ? ((LogicalProject) originalPlan).getProjects() : originalPlan.getOutput(); @@ -99,8 +236,115 @@ public class StructInfo { * Judge the source graph logical is whether the same as target * For inner join should judge only the join tables, * for other join type should also judge the join direction, it's input filter that can not be pulled up etc. - * */ - public static boolean isGraphLogicalEquals(HyperGraph source, HyperGraph target) { - return false; + */ + public static boolean isGraphLogicalEquals(StructInfo queryStructInfo, StructInfo viewStructInfo, + LogicalCompatibilityContext compatibilityContext) { + // TODO: if not inner join, should check the join graph logical equivalence + return true; + } + + private static class RelationCollector extends DefaultPlanVisitor> { + @Override + public Void visit(Plan plan, List collectedRelations) { + if (plan instanceof CatalogRelation) { + collectedRelations.add((CatalogRelation) plan); + } + return super.visit(plan, collectedRelations); + } + } + + private static class PredicateCollector extends DefaultPlanVisitor> { + @Override + public Void visit(Plan plan, Set predicates) { + if (plan instanceof LogicalFilter) { + predicates.add(((LogicalFilter) plan).getPredicate()); + } + return super.visit(plan, predicates); + } + } + + /** + * Split the plan into bottom and up, the boundary is given by context, + * the bottom contains the boundary. + */ + public static class PlanSplitter extends DefaultPlanVisitor { + @Override + public Void visit(Plan plan, PlanSplitContext context) { + if (context.getTopPlan() == null) { + context.setTopPlan(plan); + } + if (context.isBoundary(plan)) { + context.setBottomPlan(plan); + return null; + } + return super.visit(plan, context); + } + } + + /** + * Plan split context, this hold bottom and top plan, and boundary plan setting + */ + public static class PlanSplitContext { + private Plan bottomPlan; + private Plan topPlan; + private Set> boundaryPlanClazzSet; + + public PlanSplitContext(Set> boundaryPlanClazzSet) { + this.boundaryPlanClazzSet = boundaryPlanClazzSet; + } + + public Plan getBottomPlan() { + return bottomPlan; + } + + public void setBottomPlan(Plan bottomPlan) { + this.bottomPlan = bottomPlan; + } + + public Plan getTopPlan() { + return topPlan; + } + + public void setTopPlan(Plan topPlan) { + this.topPlan = topPlan; + } + + /** + * isBoundary + */ + public boolean isBoundary(Plan plan) { + for (Class boundaryPlanClazz : boundaryPlanClazzSet) { + if (boundaryPlanClazz.isAssignableFrom(plan.getClass())) { + return true; + } + } + return false; + } + } + + /** + * JoinPatternChecker + */ + public static class JoinPatternChecker extends DefaultPlanVisitor> { + @Override + public Boolean visit(Plan plan, Set requiredJoinType) { + super.visit(plan, requiredJoinType); + if (!(plan instanceof Filter) + && !(plan instanceof Project) + && !(plan instanceof CatalogRelation) + && !(plan instanceof Join)) { + return false; + } + if (plan instanceof Join) { + Join join = (Join) plan; + if (!requiredJoinType.contains(join.getJoinType())) { + return false; + } + if (!join.getOtherJoinConjuncts().isEmpty()) { + return false; + } + } + return true; + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/EquivalenceClassSetMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/EquivalenceClassSetMapping.java new file mode 100644 index 0000000000..d4ec09c24a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/EquivalenceClassSetMapping.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv.mapping; + +import org.apache.doris.nereids.rules.exploration.mv.EquivalenceClass; +import org.apache.doris.nereids.trees.expressions.SlotReference; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * EquivalenceClassSetMapping + * This will extract the equivalence class set in EquivalenceClass and mapping set in + * two different EquivalenceClass. + */ +public class EquivalenceClassSetMapping extends Mapping { + + private final Map, Set> equivalenceClassSetMap; + + public EquivalenceClassSetMapping(Map, + Set> equivalenceClassSetMap) { + this.equivalenceClassSetMap = equivalenceClassSetMap; + } + + public static EquivalenceClassSetMapping of(Map, Set> equivalenceClassSetMap) { + return new EquivalenceClassSetMapping(equivalenceClassSetMap); + } + + /** + * Generate source equivalence set map to target equivalence set + */ + public static EquivalenceClassSetMapping generate(EquivalenceClass source, EquivalenceClass target) { + + Map, Set> equivalenceClassSetMap = new HashMap<>(); + List> sourceSets = source.getEquivalenceSetList(); + List> targetSets = target.getEquivalenceSetList(); + + for (Set sourceSet : sourceSets) { + for (Set targetSet : targetSets) { + if (sourceSet.containsAll(targetSet)) { + equivalenceClassSetMap.put(sourceSet, targetSet); + } + } + } + return EquivalenceClassSetMapping.of(equivalenceClassSetMap); + } + + public Map, Set> getEquivalenceClassSetMap() { + return equivalenceClassSetMap; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java new file mode 100644 index 0000000000..7c1f06746c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv.mapping; + +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.util.ExpressionUtils; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Expression mapping, maybe one expression map to multi expression + */ +public class ExpressionMapping extends Mapping { + private final Multimap expressionMapping; + + public ExpressionMapping(Multimap expressionMapping) { + this.expressionMapping = expressionMapping; + } + + public Multimap getExpressionMapping() { + return expressionMapping; + } + + /** + * ExpressionMapping flatten + */ + public List> flattenMap() { + List>> tmpExpressionPairs = new ArrayList<>(this.expressionMapping.size()); + Map> expressionMappingMap = + expressionMapping.asMap(); + for (Map.Entry> entry + : expressionMappingMap.entrySet()) { + List> targetExpressionList = new ArrayList<>(entry.getValue().size()); + for (Expression valueExpression : entry.getValue()) { + targetExpressionList.add(Pair.of(entry.getKey(), valueExpression)); + } + tmpExpressionPairs.add(targetExpressionList); + } + List>> cartesianExpressionMap = Lists.cartesianProduct(tmpExpressionPairs); + + final List> flattenedMap = new ArrayList<>(); + for (List> listPair : cartesianExpressionMap) { + final Map expressionMap = new HashMap<>(); + listPair.forEach(pair -> expressionMap.put(pair.key(), pair.value())); + flattenedMap.add(expressionMap); + } + return flattenedMap; + } + + /**Permute the key of expression mapping. this is useful for expression rewrite, if permute key to query based + * then when expression rewrite success, we can get the mv scan expression directly. + */ + public ExpressionMapping keyPermute(SlotMapping slotMapping) { + Multimap permutedExpressionMapping = ArrayListMultimap.create(); + Map> expressionMap = + this.getExpressionMapping().asMap(); + for (Map.Entry> entry : + expressionMap.entrySet()) { + Expression replacedExpr = ExpressionUtils.replace(entry.getKey(), slotMapping.toSlotReferenceMap()); + permutedExpressionMapping.putAll(replacedExpr, entry.getValue()); + } + return new ExpressionMapping(permutedExpressionMapping); + } + + /**ExpressionMapping generate*/ + public static ExpressionMapping generate( + List sourceExpressions, + List targetExpressions) { + final Multimap expressionMultiMap = + ArrayListMultimap.create(); + for (int i = 0; i < sourceExpressions.size(); i++) { + expressionMultiMap.put(sourceExpressions.get(i), targetExpressions.get(i)); + } + return new ExpressionMapping(expressionMultiMap); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java index 520d6be9cd..3de99ea05a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java @@ -17,12 +17,13 @@ package org.apache.doris.nereids.rules.exploration.mv.mapping; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; +import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -32,19 +33,19 @@ import javax.annotation.Nullable; */ public class SlotMapping extends Mapping { - private final BiMap slotMapping; + private final BiMap relationSlotMap; + private Map slotReferenceMap; - public SlotMapping(BiMap slotMapping) { - this.slotMapping = slotMapping; + public SlotMapping(BiMap relationSlotMap) { + this.relationSlotMap = relationSlotMap; } - public BiMap getSlotBiMap() { - return slotMapping; + public BiMap getRelationSlotMap() { + return relationSlotMap; } public SlotMapping inverse() { - return slotMapping == null - ? SlotMapping.of(HashBiMap.create()) : SlotMapping.of(slotMapping.inverse()); + return SlotMapping.of(relationSlotMap.inverse()); } public static SlotMapping of(BiMap relationSlotMap) { @@ -75,10 +76,23 @@ public class SlotMapping extends Mapping { return SlotMapping.of(relationSlotMap); } + public Map toMappedSlotMap() { + return (Map) this.getRelationSlotMap(); + } + /** - * SlotMapping, getSlotMap + * SlotMapping, toSlotReferenceMap */ - public Map getSlotMap() { - return (Map) this.getSlotBiMap(); + public Map toSlotReferenceMap() { + if (this.slotReferenceMap != null) { + return this.slotReferenceMap; + } + Map slotReferenceSlotReferenceMap = new HashMap<>(); + for (Map.Entry entry : this.getRelationSlotMap().entrySet()) { + slotReferenceSlotReferenceMap.put((SlotReference) entry.getKey().getSlot(), + (SlotReference) entry.getValue().getSlot()); + } + this.slotReferenceMap = slotReferenceSlotReferenceMap; + return this.slotReferenceMap; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java index 6f937cc96d..694f061156 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableSet; import java.util.ArrayDeque; import java.util.Deque; import java.util.List; +import java.util.Set; import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; @@ -210,6 +211,19 @@ public interface TreeNode> { return (List) result.build(); } + /** + * Collect the nodes that satisfied the predicate to set. + */ + default Set collectToSet(Predicate> predicate) { + ImmutableSet.Builder> result = ImmutableSet.builder(); + foreach(node -> { + if (predicate.test(node)) { + result.add(node); + } + }); + return (Set) result.build(); + } + /** * iterate top down and test predicate if contains any instance of the classes * @param types classes array diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExprId.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExprId.java index 5e5a33fe49..77edf98353 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExprId.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExprId.java @@ -45,4 +45,14 @@ public class ExprId extends Id { public String toString() { return "" + id; } + + @Override + public boolean equals(Object obj) { + return super.equals(obj); + } + + @Override + public int hashCode() { + return super.hashCode(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/RelationId.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/RelationId.java index eb38c9a52d..23b7b69bb7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/RelationId.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/RelationId.java @@ -46,4 +46,14 @@ public class RelationId extends Id { public String toString() { return "RelationId#" + id; } + + @Override + public boolean equals(Object obj) { + return super.equals(obj); + } + + @Override + public int hashCode() { + return super.hashCode(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java index 1f8be9b5e3..44b867e688 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.ExplainOptions; import org.apache.doris.common.AnalysisException; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.glue.LogicalPlanAdapter; +import org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook; import org.apache.doris.nereids.trees.plans.Explainable; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; @@ -78,6 +79,9 @@ public class ExplainCommand extends Command implements NoForward { logicalPlanAdapter.setIsExplain(new ExplainOptions(level)); executor.setParsedStmt(logicalPlanAdapter); NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); + if (ctx.getSessionVariable().isEnableMaterializedViewRewrite()) { + planner.addHook(InitMaterializationContextHook.INSTANCE); + } planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift()); executor.setPlanner(planner); executor.checkBlockRules(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java index 54ffab7596..a3c874f637 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java @@ -39,7 +39,7 @@ public class TableCollector extends DefaultPlanVisitor shuttleExpressionWithLineage(List expressions, + Plan plan) { + return shuttleExpressionWithLineage(expressions, plan, ImmutableSet.of(), ImmutableSet.of()); + } + /** * Replace the slot in expressions with the lineage identifier from specifiedbaseTable sets or target table types * example as following: @@ -221,13 +228,18 @@ public class ExpressionUtils { ExpressionLineageReplacer.ExpressionReplaceContext replaceContext = new ExpressionLineageReplacer.ExpressionReplaceContext( - expressions.stream().map(NamedExpression.class::cast).collect(Collectors.toList()), + expressions.stream().map(Expression.class::cast).collect(Collectors.toList()), targetTypes, tableIdentifiers); plan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext); // Replace expressions by expression map - return replaceContext.getReplacedExpressions(); + List replacedExpressions = replaceContext.getReplacedExpressions(); + if (expressions.size() != replacedExpressions.size()) { + throw new NereidsException("shuttle expression fail", + new MaterializedViewException("shuttle expression fail")); + } + return replacedExpressions; } /** @@ -298,7 +310,24 @@ public class ExpressionUtils { * */ public static Expression replace(Expression expr, Map replaceMap) { - return expr.accept(ExpressionReplacer.INSTANCE, replaceMap); + return expr.accept(ExpressionReplacer.INSTANCE, ExpressionReplacerContext.of(replaceMap, false)); + } + + /** + * Replace expression node in the expression tree by `replaceMap` in top-down manner. + * if replaced, create alias + * For example. + * 
+     * input expression: a > 1
+     * replaceMap: a -> b + c
+     *
+     * output:
+     * ((b + c) as a) > 1
+     *
+ */ + public static Expression replace(Expression expr, Map replaceMap, + boolean withAlias) { + return expr.accept(ExpressionReplacer.INSTANCE, ExpressionReplacerContext.of(replaceMap, true)); } /** @@ -306,7 +335,8 @@ public class ExpressionUtils { */ public static NamedExpression replace(NamedExpression expr, Map replaceMap) { - Expression newExpr = expr.accept(ExpressionReplacer.INSTANCE, replaceMap); + Expression newExpr = expr.accept(ExpressionReplacer.INSTANCE, + ExpressionReplacerContext.of(replaceMap, false)); if (newExpr instanceof NamedExpression) { return (NamedExpression) newExpr; } else { @@ -336,18 +366,54 @@ public class ExpressionUtils { } private static class ExpressionReplacer - extends DefaultExpressionRewriter> { + extends DefaultExpressionRewriter { public static final ExpressionReplacer INSTANCE = new ExpressionReplacer(); private ExpressionReplacer() { } @Override - public Expression visit(Expression expr, Map replaceMap) { - if (replaceMap.containsKey(expr)) { - return replaceMap.get(expr); + public Expression visit(Expression expr, ExpressionReplacerContext replacerContext) { + Map replaceMap = replacerContext.getReplaceMap(); + boolean isContained = replaceMap.containsKey(expr); + if (!isContained) { + return super.visit(expr, replacerContext); } - return super.visit(expr, replaceMap); + boolean withAlias = replacerContext.isWithAlias(); + if (!withAlias) { + return replaceMap.get(expr); + } else { + Expression replacedExpression = replaceMap.get(expr); + if (replacedExpression instanceof SlotReference) { + replacedExpression = ((SlotReference) (replacedExpression)).withNullable(expr.nullable()); + } + return new Alias(((NamedExpression) expr).getExprId(), replacedExpression, + ((NamedExpression) expr).getName()); + } + } + } + + private static class ExpressionReplacerContext { + private final Map replaceMap; + private final boolean withAlias; + + private ExpressionReplacerContext(Map replaceMap, + boolean withAlias) { + this.replaceMap = replaceMap; + this.withAlias = withAlias; + } + + public static ExpressionReplacerContext of(Map replaceMap, + boolean withAlias) { + return new ExpressionReplacerContext(replaceMap, withAlias); + } + + public Map getReplaceMap() { + return replaceMap; + } + + public boolean isWithAlias() { + return withAlias; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java index 42a2c23285..89044987c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java @@ -42,6 +42,7 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.Config; import org.apache.doris.common.UserException; +import org.apache.doris.nereids.PlannerHook; import org.apache.doris.qe.CommonResultSet; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ResultSet; @@ -679,4 +680,7 @@ public class OriginalPlanner extends Planner { ResultSet resultSet = new CommonResultSet(metadata, Collections.singletonList(data)); return Optional.of(resultSet); } + + @Override + public void addHook(PlannerHook hook) {} } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java index f7899dfd33..22495e792f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java @@ -23,6 +23,7 @@ import org.apache.doris.analysis.StatementBase; import org.apache.doris.common.UserException; import org.apache.doris.common.profile.PlanTreeBuilder; import org.apache.doris.common.profile.PlanTreePrinter; +import org.apache.doris.nereids.PlannerHook; import org.apache.doris.qe.ResultSet; import org.apache.doris.thrift.TQueryOptions; @@ -123,4 +124,6 @@ public abstract class Planner { public abstract Optional handleQueryInFe(StatementBase parsedStmt); + public abstract void addHook(PlannerHook hook); + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 7fdbd06169..966635a4cb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -121,6 +121,7 @@ import org.apache.doris.nereids.exceptions.ParseException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.minidump.MinidumpUtils; import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook; import org.apache.doris.nereids.stats.StatsErrorEstimator; import org.apache.doris.nereids.trees.plans.commands.BatchInsertIntoTableCommand; import org.apache.doris.nereids.trees.plans.commands.Command; @@ -576,6 +577,9 @@ public class StmtExecutor { } // create plan planner = new NereidsPlanner(statementContext); + if (context.getSessionVariable().isEnableMaterializedViewRewrite()) { + planner.addHook(InitMaterializationContextHook.INSTANCE); + } try { planner.plan(parsedStmt, context.getSessionVariable().toThrift()); checkBlockRules(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java index 39f6033b4a..e032285243 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java @@ -288,7 +288,7 @@ public class MappingTest extends TestWithFeService { SlotMapping slotMapping = SlotMapping.generate(relationMapping); Assertions.assertNotNull(slotMapping); BiMap generatedSlotMapping = HashBiMap.create(); - slotMapping.getSlotBiMap().forEach((key, value) -> + slotMapping.getRelationSlotMap().forEach((key, value) -> generatedSlotMapping.put(key.getExprId(), value.getExprId()) ); Assertions.assertEquals(generatedSlotMapping, expectSlotMapping); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/PredicatesSplitterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/PredicatesSplitterTest.java index 4a8f2e981e..7c3055b9ce 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/PredicatesSplitterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/PredicatesSplitterTest.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -68,21 +69,21 @@ public class PredicatesSplitterTest extends ExpressionRewriteTestHelper { Expression equalExpression = replaceUnboundSlot(PARSER.parseExpression(expectedEqualExpr), mem); Assertions.assertEquals(equalExpression, splitPredicate.getEqualPredicate()); } else { - Assertions.assertNull(splitPredicate.getEqualPredicate()); + Assertions.assertEquals(splitPredicate.getEqualPredicate(), BooleanLiteral.TRUE); } if (!StringUtils.isEmpty(expectedRangeExpr)) { Expression rangeExpression = replaceUnboundSlot(PARSER.parseExpression(expectedRangeExpr), mem); Assertions.assertEquals(rangeExpression, splitPredicate.getRangePredicate()); } else { - Assertions.assertNull(splitPredicate.getRangePredicate()); + Assertions.assertEquals(splitPredicate.getRangePredicate(), BooleanLiteral.TRUE); } if (!StringUtils.isEmpty(expectedResidualExpr)) { Expression residualExpression = replaceUnboundSlot(PARSER.parseExpression(expectedResidualExpr), mem); Assertions.assertEquals(residualExpression, splitPredicate.getResidualPredicate()); } else { - Assertions.assertNull(splitPredicate.getResidualPredicate()); + Assertions.assertEquals(splitPredicate.getResidualPredicate(), BooleanLiteral.TRUE); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java index 39aede7a74..058a2de2f0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java @@ -165,7 +165,7 @@ public class PlanChecker { public PlanChecker applyTopDown(List rule) { Rewriter.getWholeTreeRewriterWithCustomJobs(cascadesContext, - ImmutableList.of(new RootPlanTreeRewriteJob(rule, PlanTreeRewriteTopDownJob::new, true))) + ImmutableList.of(new RootPlanTreeRewriteJob(rule, PlanTreeRewriteTopDownJob::new, true))) .execute(); cascadesContext.toMemo(); MemoValidator.validate(cascadesContext.getMemo()); diff --git a/regression-test/data/nereids_rules_p0/mv/inner_join.out b/regression-test/data/nereids_rules_p0/mv/inner_join.out new file mode 100644 index 0000000000..9997485376 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/inner_join.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query1_0 -- + +-- !query1_1 -- + +-- !query1_2 -- + +-- !query1_3 -- + diff --git a/regression-test/suites/nereids_rules_p0/mv/inner_join.groovy b/regression-test/suites/nereids_rules_p0/mv/inner_join.groovy new file mode 100644 index 0000000000..9d0bc62f05 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/inner_join.groovy @@ -0,0 +1,165 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("inner_join") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql "SET enable_materialized_view_rewrite=true" + sql "SET enable_nereids_timeout = false" + + sql """ + drop table if exists orders + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders ( + O_ORDERKEY INTEGER NOT NULL, + O_CUSTKEY INTEGER NOT NULL, + O_ORDERSTATUS CHAR(1) NOT NULL, + O_TOTALPRICE DECIMALV3(15,2) NOT NULL, + O_ORDERDATE DATE NOT NULL, + O_ORDERPRIORITY CHAR(15) NOT NULL, + O_CLERK CHAR(15) NOT NULL, + O_SHIPPRIORITY INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL + ) + DUPLICATE KEY(O_ORDERKEY, O_CUSTKEY) + DISTRIBUTED BY HASH(O_ORDERKEY) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists lineitem + """ + + sql""" + CREATE TABLE IF NOT EXISTS lineitem ( + L_ORDERKEY INTEGER NOT NULL, + L_PARTKEY INTEGER NOT NULL, + L_SUPPKEY INTEGER NOT NULL, + L_LINENUMBER INTEGER NOT NULL, + L_QUANTITY DECIMALV3(15,2) NOT NULL, + L_EXTENDEDPRICE DECIMALV3(15,2) NOT NULL, + L_DISCOUNT DECIMALV3(15,2) NOT NULL, + L_TAX DECIMALV3(15,2) NOT NULL, + L_RETURNFLAG CHAR(1) NOT NULL, + L_LINESTATUS CHAR(1) NOT NULL, + L_SHIPDATE DATE NOT NULL, + L_COMMITDATE DATE NOT NULL, + L_RECEIPTDATE DATE NOT NULL, + L_SHIPINSTRUCT CHAR(25) NOT NULL, + L_SHIPMODE CHAR(10) NOT NULL, + L_COMMENT VARCHAR(44) NOT NULL + ) + DUPLICATE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER) + DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists partsupp + """ + + sql """ + CREATE TABLE IF NOT EXISTS partsupp ( + PS_PARTKEY INTEGER NOT NULL, + PS_SUPPKEY INTEGER NOT NULL, + PS_AVAILQTY INTEGER NOT NULL, + PS_SUPPLYCOST DECIMALV3(15,2) NOT NULL, + PS_COMMENT VARCHAR(199) NOT NULL + ) + DUPLICATE KEY(PS_PARTKEY, PS_SUPPKEY) + DISTRIBUTED BY HASH(PS_PARTKEY) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + def check_rewrite = { mv_sql, query_sql, mv_name -> + + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}""" + sql""" + CREATE MATERIALIZED VIEW ${mv_name} + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS ${mv_sql} + """ + + def job_name = getJobName(db, mv_name); + waitingMTMVTaskFinished(job_name) + explain { + sql("${query_sql}") + contains "(${mv_name})" + } + } + + // select + from + inner join + def mv1_0 = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY " + + "from lineitem " + + "inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + def query1_0 = "select lineitem.L_LINENUMBER " + + "from lineitem " + + "inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + check_rewrite(mv1_0, query1_0, "mv1_0") + order_qt_query1_0 "${query1_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" + + + def mv1_1 = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY, partsupp.PS_AVAILQTY " + + "from lineitem " + + "inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + + "inner join partsupp on lineitem.L_PARTKEY = partsupp.PS_PARTKEY " + + "and lineitem.L_SUPPKEY = partsupp.PS_SUPPKEY" + def query1_1 = "select lineitem.L_LINENUMBER " + + "from lineitem " + + "inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + + "inner join partsupp on lineitem.L_PARTKEY = partsupp.PS_PARTKEY " + + "and lineitem.L_SUPPKEY = partsupp.PS_SUPPKEY" + check_rewrite(mv1_1, query1_1, "mv1_1") + order_qt_query1_1 "${query1_1}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_1""" + + + def mv1_2 = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY " + + "from orders " + + "inner join lineitem on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + def query1_2 = "select lineitem.L_LINENUMBER " + + "from lineitem " + + "inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + check_rewrite(mv1_2, query1_2, "mv1_2") + order_qt_query1_2 "${query1_2}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_2""" + + // select + from + inner join + filter + def mv1_3 = "select lineitem.L_LINENUMBER, orders.O_CUSTKEY " + + "from orders " + + "inner join lineitem on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + def query1_3 = "select lineitem.L_LINENUMBER " + + "from lineitem " + + "inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " + + "where lineitem.L_LINENUMBER > 10" + check_rewrite(mv1_3, query1_3, "mv1_3") + order_qt_query1_3 "${query1_3}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_3""" +}