database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](nereids) Support inner join query rewrite by materialized view (#27922)

Browse Source 
Work in process. Support inner join query rewrite by materialized view in some scene.
Such as an exmple as following:

> mv = "select  lineitem.L_LINENUMBER, orders.O_CUSTKEY " +
>             "from orders " +
>             "inner join lineitem on lineitem.L_ORDERKEY = orders.O_ORDERKEY "
>     query = "select lineitem.L_LINENUMBER " +
>             "from lineitem " +
>             "inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY "
This commit is contained in:
seawinde
2023-12-07 20:29:51 +08:00
committed by GitHub
parent f37215a32a
commit be81eb1a9b
34 changed files with 1615 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
View File

@ -30,6 +30,7 @@ import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState;
import org.apache.doris.mtmv.MTMVRefreshInfo;
import org.apache.doris.mtmv.MTMVRelation;
import org.apache.doris.mtmv.MTMVStatus;
import org.apache.doris.mtmv.MVCache;
import org.apache.doris.persist.gson.GsonUtils;
import com.google.gson.annotations.SerializedName;
@ -61,6 +62,8 @@ public class MTMV extends OlapTable {
private Map<String, String> mvProperties;
@SerializedName("r")
private MTMVRelation relation;
// Should update after every fresh
private MVCache mvCache;
// For deserialization
public MTMV() {
@ -116,6 +119,14 @@ public class MTMV extends OlapTable {
return relation;
}
public MVCache getMvCache() {
return mvCache;
}
public void setMvCache(MVCache mvCache) {
this.mvCache = mvCache;
}
public MTMVRefreshInfo alterRefreshInfo(MTMVRefreshInfo newRefreshInfo) {
return refreshInfo.updateNotNull(newRefreshInfo);
}

45
fe/fe-core/src/main/java/org/apache/doris/common/MaterializedViewException.java Normal file
View File

@ -0,0 +1,45 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.common;
/**
* MaterializedViewException
*/
public class MaterializedViewException extends UserException {
public MaterializedViewException(String msg, Throwable cause) {
super(msg, cause);
}
public MaterializedViewException(Throwable cause) {
super(cause);
}
public MaterializedViewException(String msg, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(msg, cause, enableSuppression, writableStackTrace);
}
public MaterializedViewException(String msg) {
super(msg);
}
public MaterializedViewException(InternalErrorCode errCode, String msg) {
super(errCode, msg);
}
}

6
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCacheManager.java
View File

@ -48,6 +48,7 @@ import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollecto
import org.apache.doris.persist.AlterMTMV;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.collections.CollectionUtils;
@ -70,6 +71,11 @@ public class MTMVCacheManager implements MTMVHookService {
return tableMTMVs.get(table);
}
// TODO Implement the method which getting materialized view by tables
public List<MTMV> getAvailableMaterializedView(List<BaseTableInfo> tables) {
return ImmutableList.of();
}
public boolean isAvailableMTMV(MTMV mtmv, ConnectContext ctx) throws AnalysisException, DdlException {
// check session variable if enable rewrite
if (!ctx.getSessionVariable().isEnableMvRewrite()) {

82
fe/fe-core/src/main/java/org/apache/doris/mtmv/MVCache.java Normal file
View File

@ -0,0 +1,82 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.mtmv;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.OriginStatement;
import java.util.List;
import java.util.stream.Collectors;
/**
* The cache for materialized view cache
*/
public class MVCache {
// the materialized view plan which should be optimized by the same rules to query
private final Plan logicalPlan;
// this should be shuttle expression with lineage
private final List<NamedExpression> mvOutputExpressions;
public MVCache(MTMV materializedView, Plan logicalPlan, List<NamedExpression> mvOutputExpressions) {
this.logicalPlan = logicalPlan;
this.mvOutputExpressions = mvOutputExpressions;
}
public Plan getLogicalPlan() {
return logicalPlan;
}
public List<NamedExpression> getMvOutputExpressions() {
return mvOutputExpressions;
}
public MVCache(Plan logicalPlan, List<NamedExpression> mvOutputExpressions) {
this.logicalPlan = logicalPlan;
this.mvOutputExpressions = mvOutputExpressions;
}
public static MVCache from(MTMV mtmv, ConnectContext connectContext) {
LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(mtmv.getQuerySql());
// TODO: connect context set current db when create mv by use database
StatementContext mvSqlStatementContext = new StatementContext(connectContext,
new OriginStatement(mtmv.getQuerySql(), 0));
NereidsPlanner planner = new NereidsPlanner(mvSqlStatementContext);
Plan mvRewrittenPlan =
planner.plan(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN);
Plan mvPlan = mvRewrittenPlan instanceof LogicalResultSink
? (Plan) ((LogicalResultSink) mvRewrittenPlan).child() : mvRewrittenPlan;
// use rewritten plan output expression currently, if expression rewrite fail,
// consider to use the analyzed plan for output expressions only
List<NamedExpression> mvOutputExpressions = mvRewrittenPlan.getExpressions().stream()
.map(NamedExpression.class::cast)
.collect(Collectors.toList());
return new MVCache(mvPlan, mvOutputExpressions);
}
}

14
fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
View File

@ -88,6 +88,7 @@ public class NereidsPlanner extends Planner {
private PhysicalPlan physicalPlan;
// The cost of optimized plan
private double cost = 0;
private List<PlannerHook> hooks = new ArrayList<>();
public NereidsPlanner(StatementContext statementContext) {
this.statementContext = statementContext;
@ -260,15 +261,12 @@ public class NereidsPlanner extends Planner {
if (statementContext.getConnectContext().getTables() != null) {
cascadesContext.setTables(statementContext.getConnectContext().getTables());
}
if (statementContext.getConnectContext().getSessionVariable().isEnableMaterializedViewRewrite()) {
// TODO Pre handle materialized view to materializationContext and
// call cascadesContext.addMaterializationContext() to add it
}
}
private void analyze() {
LOG.debug("Start analyze plan");
cascadesContext.newAnalyzer().analyze();
getHooks().forEach(hook -> hook.afterAnalyze(this));
NereidsTracer.logImportantTime("EndAnalyzePlan");
LOG.debug("End analyze plan");
}
@ -525,4 +523,12 @@ public class NereidsPlanner extends Planner {
public PhysicalPlan getPhysicalPlan() {
return physicalPlan;
}
public List<PlannerHook> getHooks() {
return hooks;
}
public void addHook(PlannerHook hook) {
this.hooks.add(hook);
}
}

38
fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerHook.java Normal file
View File

@ -0,0 +1,38 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids;
/**
* optimize plan process has some phase, such as analyze, rewrite, optimize, generate physical plan
* and so on, this hook give a chance to do something in the planning process.
* For example: after analyze plan when query or explain, we should generate materialization context.
*/
public interface PlannerHook {
/**
* the hook before analyze
*/
default void beforeAnalyze(NereidsPlanner planner) {
}
/**
* the hook after analyze
*/
default void afterAnalyze(NereidsPlanner planner) {
}
}

6
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java
View File

@ -22,7 +22,9 @@ import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.JobType;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.qe.ConnectContext;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@ -46,7 +48,9 @@ public class OptimizeGroupExpressionJob extends Job {
countJobExecutionTimesOfGroupExpressions(groupExpression);
List<Rule> implementationRules = getRuleSet().getImplementationRules();
List<Rule> explorationRules = getExplorationRules();
if (context.getCascadesContext().getConnectContext().getSessionVariable().isEnableMaterializedViewRewrite()) {
ConnectContext connectContext = context.getCascadesContext().getConnectContext();
if (connectContext.getSessionVariable().isEnableMaterializedViewRewrite()) {
explorationRules = new ArrayList<>(explorationRules);
explorationRules.addAll(getRuleSet().getMaterializedViewRules());
}

14
fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
View File

@ -76,7 +76,7 @@ public class Group {
private int chosenGroupExpressionId = -1;
private Optional<StructInfo> structInfo = Optional.empty();
private List<StructInfo> structInfos = new ArrayList<>();
/**
* Constructor for Group.
@ -541,11 +541,15 @@ public class Group {
return TreeStringUtils.treeString(this, toString, getChildren, getExtraPlans, displayExtraPlan);
}
public Optional<StructInfo> getStructInfo() {
return structInfo;
public List<StructInfo> getStructInfos() {
return structInfos;
}
public void setStructInfo(StructInfo structInfo) {
this.structInfo = Optional.ofNullable(structInfo);
public void addStructInfo(StructInfo structInfo) {
this.structInfos.add(structInfo);
}
public void addStructInfo(List<StructInfo> structInfos) {
this.structInfos.addAll(structInfos);
}
}

71
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java
View File

@ -17,47 +17,86 @@
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.Edge;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.AbstractNode;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.Sets;
import java.util.HashSet;
import java.util.List;
import java.util.stream.Collectors;
/**
* AbstractMaterializedViewJoinRule
* This is responsible for common join rewriting
*/
public abstract class AbstractMaterializedViewJoinRule extends AbstractMaterializedViewRule {
private static final HashSet<JoinType> SUPPORTED_JOIN_TYPE_SET =
Sets.newHashSet(JoinType.INNER_JOIN, JoinType.LEFT_OUTER_JOIN);
@Override
protected Plan rewriteQueryByView(MatchMode matchMode,
StructInfo queryStructInfo,
StructInfo viewStructInfo,
RelationMapping queryToViewTableMappings,
Plan tempRewritedPlan) {
SlotMapping queryToViewSlotMappings,
Plan tempRewritedPlan,
MaterializationContext materializationContext) {
// Rewrite top projects, represent the query projects by view
List<NamedExpression> expressions = rewriteExpression(
List<? extends Expression> queryShuttleExpression = ExpressionUtils.shuttleExpressionWithLineage(
queryStructInfo.getExpressions(),
queryStructInfo,
viewStructInfo,
queryToViewTableMappings,
tempRewritedPlan
queryStructInfo.getOriginalPlan());
// Rewrite top projects, represent the query projects by view
List<Expression> expressionsRewritten = rewriteExpression(
queryShuttleExpression,
materializationContext.getViewExpressionIndexMapping(),
queryToViewSlotMappings
);
// Can not rewrite, bail out
if (expressions == null) {
if (expressionsRewritten == null
|| expressionsRewritten.stream().anyMatch(expr -> !(expr instanceof NamedExpression))) {
return null;
}
return new LogicalProject<>(expressions, tempRewritedPlan);
// record the group id in materializationContext, and when rewrite again in
// the same group, bail out quickly.
if (queryStructInfo.getOriginalPlan().getGroupExpression().isPresent()) {
materializationContext.addMatchedGroup(
queryStructInfo.getOriginalPlan().getGroupExpression().get().getOwnerGroup().getGroupId());
}
return new LogicalProject<>(
expressionsRewritten.stream().map(NamedExpression.class::cast).collect(Collectors.toList()),
tempRewritedPlan);
}
// Check join is whether valid or not. Support join's input can not contain aggregate
// Only support project, filter, join, logical relation node and
// join condition should be slot reference equals currently
/**
* Check join is whether valid or not. Support join's input can not contain aggregate
* Only support project, filter, join, logical relation node and
* join condition should be slot reference equals currently
*/
@Override
protected boolean checkPattern(StructInfo structInfo) {
// TODO Should get struct info from hyper graph and check
return false;
HyperGraph hyperGraph = structInfo.getHyperGraph();
for (AbstractNode node : hyperGraph.getNodes()) {
StructInfoNode structInfoNode = (StructInfoNode) node;
if (!structInfoNode.getPlan().accept(StructInfo.JOIN_PATTERN_CHECKER,
SUPPORTED_JOIN_TYPE_SET)) {
return false;
}
}
for (Edge edge : hyperGraph.getEdges()) {
if (!edge.getJoin().accept(StructInfo.JOIN_PATTERN_CHECKER,
SUPPORTED_JOIN_TYPE_SET)) {
return false;
}
}
return true;
}
}

257
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View File

@ -19,23 +19,31 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate;
import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionIndexMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.EquivalenceClassSetMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
/**
@ -53,54 +61,81 @@ public abstract class AbstractMaterializedViewRule {
if (materializationContexts.isEmpty()) {
return rewriteResults;
}
StructInfo queryStructInfo = extractStructInfo(queryPlan, cascadesContext);
// Check query queryPlan
List<StructInfo> queryStructInfos = extractStructInfo(queryPlan, cascadesContext);
// TODO Just Check query queryPlan firstly, support multi later.
StructInfo queryStructInfo = queryStructInfos.get(0);
if (!checkPattern(queryStructInfo)) {
return rewriteResults;
}
for (MaterializationContext materializationContext : materializationContexts) {
Plan mvPlan = materializationContext.getMvPlan();
StructInfo viewStructInfo = extractStructInfo(mvPlan, cascadesContext);
// already rewrite, bail out
if (queryPlan.getGroupExpression().isPresent()
&& materializationContext.alreadyRewrite(
queryPlan.getGroupExpression().get().getOwnerGroup().getGroupId())) {
continue;
}
Plan mvPlan = materializationContext.getMtmv().getMvCache().getLogicalPlan();
List<StructInfo> viewStructInfos = extractStructInfo(mvPlan, cascadesContext);
if (viewStructInfos.size() > 1) {
// view struct info should only have one
return rewriteResults;
}
StructInfo viewStructInfo = viewStructInfos.get(0);
if (!checkPattern(viewStructInfo)) {
continue;
}
if (!StructInfo.isGraphLogicalEquals(queryStructInfo.getHyperGraph(), viewStructInfo.getHyperGraph())) {
continue;
}
MatchMode matchMode = decideMatchMode(queryStructInfo.getRelations(), viewStructInfo.getRelations());
if (MatchMode.NOT_MATCH == matchMode) {
if (MatchMode.COMPLETE != matchMode) {
continue;
}
List<RelationMapping> queryToViewTableMappings =
RelationMapping.generate(queryStructInfo.getRelations(), viewStructInfo.getRelations());
// if any relation in query and view can not map, bail out.
if (queryToViewTableMappings == null) {
return rewriteResults;
}
for (RelationMapping queryToViewTableMapping : queryToViewTableMappings) {
SlotMapping queryToViewSlotMapping = SlotMapping.generate(queryToViewTableMapping);
if (queryToViewSlotMapping == null) {
continue;
}
LogicalCompatibilityContext compatibilityContext =
LogicalCompatibilityContext.from(queryToViewTableMapping, queryToViewSlotMapping,
queryStructInfo, viewStructInfo);
// todo outer join compatibility check
if (!StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo, compatibilityContext)) {
continue;
}
SplitPredicate compensatePredicates = predicatesCompensate(queryStructInfo, viewStructInfo,
queryToViewTableMapping);
queryToViewSlotMapping);
// Can not compensate, bail out
if (compensatePredicates == null || compensatePredicates.isEmpty()) {
if (compensatePredicates.isEmpty()) {
continue;
}
Plan rewritedPlan;
Plan mvScan = materializationContext.getScanPlan();
Plan mvScan = materializationContext.getMvScanPlan();
if (compensatePredicates.isAlwaysTrue()) {
rewritedPlan = mvScan;
} else {
// Try to rewrite compensate predicates by using mv scan
List<NamedExpression> rewriteCompensatePredicates = rewriteExpression(
List<Expression> rewriteCompensatePredicates = rewriteExpression(
compensatePredicates.toList(),
queryStructInfo,
viewStructInfo,
queryToViewTableMapping,
mvScan);
materializationContext.getViewExpressionIndexMapping(),
queryToViewSlotMapping);
if (rewriteCompensatePredicates.isEmpty()) {
continue;
}
rewritedPlan = new LogicalFilter<>(Sets.newHashSet(rewriteCompensatePredicates), mvScan);
}
// Rewrite query by view
rewritedPlan = rewriteQueryByView(matchMode, queryStructInfo, viewStructInfo,
queryToViewTableMapping, rewritedPlan);
rewritedPlan = rewriteQueryByView(matchMode,
queryStructInfo,
viewStructInfo,
queryToViewSlotMapping,
rewritedPlan,
materializationContext);
if (rewritedPlan == null) {
continue;
}
@ -110,22 +145,25 @@ public abstract class AbstractMaterializedViewRule {
return rewriteResults;
}
/**Rewrite query by view, for aggregate or join rewriting should be different inherit class implementation*/
/**
* Rewrite query by view, for aggregate or join rewriting should be different inherit class implementation
*/
protected Plan rewriteQueryByView(MatchMode matchMode,
StructInfo queryStructInfo,
StructInfo viewStructInfo,
RelationMapping queryToViewTableMappings,
Plan tempRewritedPlan) {
SlotMapping queryToViewSlotMappings,
Plan tempRewritedPlan,
MaterializationContext materializationContext) {
return tempRewritedPlan;
}
/**Use target output expression to represent the source expression*/
protected List<NamedExpression> rewriteExpression(List<? extends Expression> sourceExpressions,
StructInfo sourceStructInfo,
StructInfo targetStructInfo,
RelationMapping sourceToTargetMapping,
Plan targetScanNode) {
// TODO represent the sourceExpressions by using target scan node
/**
* Use target output expression to represent the source expression
*/
protected List<Expression> rewriteExpression(
List<? extends Expression> sourceExpressionsToWrite,
ExpressionMapping mvExprToMvScanExprMapping,
SlotMapping sourceToTargetMapping) {
// Firstly, rewrite the target plan output expression using query with inverse mapping
// then try to use the mv expression to represent the query. if any of source expressions
// can not be represented by mv, return null
@ -138,18 +176,34 @@ public abstract class AbstractMaterializedViewRule {
// transform source to:
// project(slot 2, 1)
// target
List<? extends Expression> targetTopExpressions = targetStructInfo.getExpressions();
List<? extends Expression> shuttledTargetExpressions = ExpressionUtils.shuttleExpressionWithLineage(
targetTopExpressions, targetStructInfo.getOriginalPlan(), Sets.newHashSet(), Sets.newHashSet());
SlotMapping sourceToTargetSlotMapping = SlotMapping.generate(sourceToTargetMapping);
// mv sql plan expressions transform to query based
List<? extends Expression> queryBasedExpressions = ExpressionUtils.replace(
shuttledTargetExpressions.stream().map(Expression.class::cast).collect(Collectors.toList()),
sourceToTargetSlotMapping.inverse().getSlotMap());
// mv sql query based expression and index mapping
ExpressionIndexMapping.generate(queryBasedExpressions);
// TODO visit source expression and replace the expression with expressionIndexMapping
return ImmutableList.of();
// generate mvSql to mvScan mvExprToMvScanExprMapping, and change mv sql expression to query based
ExpressionMapping mvExprToMvScanExprMappingKeySourceBased =
mvExprToMvScanExprMapping.keyPermute(sourceToTargetMapping.inverse());
List<Map<? extends Expression, ? extends Expression>> flattenExpressionMapping =
mvExprToMvScanExprMappingKeySourceBased.flattenMap();
// view to view scan expression is 1:1 so get first element
Map<? extends Expression, ? extends Expression> mvSqlToMvScanMappingQueryBased =
flattenExpressionMapping.get(0);
List<Expression> rewrittenExpressions = new ArrayList<>();
for (Expression expressionToRewrite : sourceExpressionsToWrite) {
if (expressionToRewrite instanceof Literal) {
rewrittenExpressions.add(expressionToRewrite);
continue;
}
final Set<Object> slotsToRewrite =
expressionToRewrite.collectToSet(expression -> expression instanceof Slot);
boolean wiAlias = expressionToRewrite instanceof NamedExpression;
Expression replacedExpression = ExpressionUtils.replace(expressionToRewrite,
mvSqlToMvScanMappingQueryBased,
wiAlias);
if (replacedExpression.anyMatch(slotsToRewrite::contains)) {
// if contains any slot to rewrite, which means can not be rewritten by target, bail out
return null;
}
rewrittenExpressions.add(replacedExpression);
}
return rewrittenExpressions;
}
/**
@ -161,21 +215,109 @@ public abstract class AbstractMaterializedViewRule {
protected SplitPredicate predicatesCompensate(
StructInfo queryStructInfo,
StructInfo viewStructInfo,
RelationMapping queryToViewTableMapping
SlotMapping queryToViewSlotMapping
) {
// TODO Equal predicate compensate
EquivalenceClass queryEquivalenceClass = queryStructInfo.getEquivalenceClass();
EquivalenceClass viewEquivalenceClass = viewStructInfo.getEquivalenceClass();
// viewEquivalenceClass to query based
Map<SlotReference, SlotReference> viewToQuerySlotMapping = queryToViewSlotMapping.inverse()
.toSlotReferenceMap();
EquivalenceClass viewEquivalenceClassQueryBased = viewEquivalenceClass.permute(viewToQuerySlotMapping);
if (viewEquivalenceClassQueryBased == null) {
return SplitPredicate.empty();
}
final List<Expression> equalCompensateConjunctions = new ArrayList<>();
if (queryEquivalenceClass.isEmpty() && viewEquivalenceClass.isEmpty()) {
equalCompensateConjunctions.add(BooleanLiteral.of(true));
}
if (queryEquivalenceClass.isEmpty()
&& !viewEquivalenceClass.isEmpty()) {
return null;
return SplitPredicate.empty();
}
// TODO range predicates and residual predicates compensate
return SplitPredicate.empty();
EquivalenceClassSetMapping queryToViewEquivalenceMapping =
EquivalenceClassSetMapping.generate(queryEquivalenceClass, viewEquivalenceClassQueryBased);
// can not map all target equivalence class, can not compensate
if (queryToViewEquivalenceMapping.getEquivalenceClassSetMap().size()
< viewEquivalenceClass.getEquivalenceSetList().size()) {
return SplitPredicate.empty();
}
// do equal compensate
Set<Set<SlotReference>> mappedQueryEquivalenceSet =
queryToViewEquivalenceMapping.getEquivalenceClassSetMap().keySet();
queryEquivalenceClass.getEquivalenceSetList().forEach(
queryEquivalenceSet -> {
// compensate the equivalence in query but not in view
if (!mappedQueryEquivalenceSet.contains(queryEquivalenceSet)) {
Iterator<SlotReference> iterator = queryEquivalenceSet.iterator();
SlotReference first = iterator.next();
while (iterator.hasNext()) {
Expression equals = new EqualTo(first, iterator.next());
equalCompensateConjunctions.add(equals);
}
} else {
// compensate the equivalence both in query and view, but query has more equivalence
Set<SlotReference> viewEquivalenceSet =
queryToViewEquivalenceMapping.getEquivalenceClassSetMap().get(queryEquivalenceSet);
Set<SlotReference> copiedQueryEquivalenceSet = new HashSet<>(queryEquivalenceSet);
copiedQueryEquivalenceSet.removeAll(viewEquivalenceSet);
SlotReference first = viewEquivalenceSet.iterator().next();
for (SlotReference slotReference : copiedQueryEquivalenceSet) {
Expression equals = new EqualTo(first, slotReference);
equalCompensateConjunctions.add(equals);
}
}
}
);
// TODO range predicates and residual predicates compensate, Simplify implementation.
SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate();
SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate();
// range compensate
List<Expression> rangeCompensate = new ArrayList<>();
Expression queryRangePredicate = querySplitPredicate.getRangePredicate();
Expression viewRangePredicate = viewSplitPredicate.getRangePredicate();
Expression viewRangePredicateQueryBased =
ExpressionUtils.replace(viewRangePredicate, viewToQuerySlotMapping);
Set<Expression> queryRangeSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(queryRangePredicate));
Set<Expression> viewRangeQueryBasedSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(viewRangePredicateQueryBased));
// query range predicate can not contain all view range predicate when view have range predicate, bail out
if (!viewRangePredicateQueryBased.equals(BooleanLiteral.TRUE)
&& !queryRangeSet.containsAll(viewRangeQueryBasedSet)) {
return SplitPredicate.empty();
}
queryRangeSet.removeAll(viewRangeQueryBasedSet);
rangeCompensate.addAll(queryRangeSet);
// residual compensate
List<Expression> residualCompensate = new ArrayList<>();
Expression queryResidualPredicate = querySplitPredicate.getResidualPredicate();
Expression viewResidualPredicate = viewSplitPredicate.getResidualPredicate();
Expression viewResidualPredicateQueryBased =
ExpressionUtils.replace(viewResidualPredicate, viewToQuerySlotMapping);
Set<Expression> queryResidualSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(queryResidualPredicate));
Set<Expression> viewResidualQueryBasedSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(viewResidualPredicateQueryBased));
// query residual predicate can not contain all view residual predicate when view have residual predicate,
// bail out
if (!viewResidualPredicateQueryBased.equals(BooleanLiteral.TRUE)
&& !queryResidualSet.containsAll(viewResidualQueryBasedSet)) {
return SplitPredicate.empty();
}
queryResidualSet.removeAll(viewResidualQueryBasedSet);
residualCompensate.addAll(queryResidualSet);
return SplitPredicate.of(ExpressionUtils.and(equalCompensateConjunctions),
rangeCompensate.isEmpty() ? BooleanLiteral.of(true) : ExpressionUtils.and(rangeCompensate),
residualCompensate.isEmpty() ? BooleanLiteral.of(true) : ExpressionUtils.and(residualCompensate));
}
/**
* Decide the match mode
*
* @see MatchMode
*/
private MatchMode decideMatchMode(List<CatalogRelation> queryRelations, List<CatalogRelation> viewRelations) {
@ -205,20 +347,17 @@ public abstract class AbstractMaterializedViewRule {
/**
* Extract struct info from plan, support to get struct info from logical plan or plan in group.
*/
protected StructInfo extractStructInfo(Plan plan, CascadesContext cascadesContext) {
protected List<StructInfo> extractStructInfo(Plan plan, CascadesContext cascadesContext) {
if (plan.getGroupExpression().isPresent()
&& plan.getGroupExpression().get().getOwnerGroup().getStructInfo().isPresent()) {
Group belongGroup = plan.getGroupExpression().get().getOwnerGroup();
return belongGroup.getStructInfo().get();
&& !plan.getGroupExpression().get().getOwnerGroup().getStructInfos().isEmpty()) {
return plan.getGroupExpression().get().getOwnerGroup().getStructInfos();
} else {
// TODO build graph from plan and extract struct from graph and set to group if exist
// Should get structInfo from hyper graph and add into current group
StructInfo structInfo = StructInfo.of(plan);
// build struct info and add them to current group
List<StructInfo> structInfos = StructInfo.of(plan);
if (plan.getGroupExpression().isPresent()) {
plan.getGroupExpression().get().getOwnerGroup().setStructInfo(structInfo);
plan.getGroupExpression().get().getOwnerGroup().addStructInfo(structInfos);
}
return structInfo;
return structInfos;
}
}
@ -229,7 +368,7 @@ public abstract class AbstractMaterializedViewRule {
if (structInfo.getRelations().isEmpty()) {
return false;
}
return false;
return true;
}
/**

64
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java
View File

@ -20,6 +20,8 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
@ -31,11 +33,24 @@ import java.util.Set;
*/
public class EquivalenceClass {
private final Map<SlotReference, Set<SlotReference>> equivalenceSlotMap = new LinkedHashMap<>();
/**
* eg: column a = b
* this will be
* {
* a: [a, b],
* b: [a, b]
* }
*/
private Map<SlotReference, Set<SlotReference>> equivalenceSlotMap = new LinkedHashMap<>();
private List<Set<SlotReference>> equivalenceSlotList;
public EquivalenceClass() {
}
public EquivalenceClass(Map<SlotReference, Set<SlotReference>> equivalenceSlotMap) {
this.equivalenceSlotMap = equivalenceSlotMap;
}
/**
* EquivalenceClass
*/
@ -82,11 +97,48 @@ public class EquivalenceClass {
}
/**
* EquivalenceClass
* EquivalenceClass permute
*/
public List<Set<SlotReference>> getEquivalenceValues() {
List<Set<SlotReference>> values = new ArrayList<>();
equivalenceSlotMap.values().forEach(each -> values.add(each));
return values;
public EquivalenceClass permute(Map<SlotReference, SlotReference> mapping) {
Map<SlotReference, Set<SlotReference>> permutedEquivalenceSlotMap = new HashMap<>();
for (Map.Entry<SlotReference, Set<SlotReference>> slotReferenceSetEntry : equivalenceSlotMap.entrySet()) {
SlotReference mappedSlotReferenceKey = mapping.get(slotReferenceSetEntry.getKey());
if (mappedSlotReferenceKey == null) {
// can not permute then need to return null
return null;
}
Set<SlotReference> equivalenceValueSet = slotReferenceSetEntry.getValue();
final Set<SlotReference> mappedSlotReferenceSet = new HashSet<>();
for (SlotReference target : equivalenceValueSet) {
SlotReference mappedSlotReferenceValue = mapping.get(target);
if (mappedSlotReferenceValue == null) {
return null;
}
mappedSlotReferenceSet.add(mappedSlotReferenceValue);
}
permutedEquivalenceSlotMap.put(mappedSlotReferenceKey, mappedSlotReferenceSet);
}
return new EquivalenceClass(permutedEquivalenceSlotMap);
}
/**
* Return the list of equivalence set, remove duplicate
*/
public List<Set<SlotReference>> getEquivalenceSetList() {
if (equivalenceSlotList != null) {
return equivalenceSlotList;
}
List<Set<SlotReference>> equivalenceSets = new ArrayList<>();
Set<Set<SlotReference>> visited = new HashSet<>();
equivalenceSlotMap.values().forEach(slotSet -> {
if (!visited.contains(slotSet)) {
equivalenceSets.add(slotSet);
}
visited.add(slotSet);
});
this.equivalenceSlotList = equivalenceSets;
return this.equivalenceSlotList;
}
}

111
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java Normal file
View File

@ -0,0 +1,111 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.mtmv.MTMVCacheManager;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.PlannerHook;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.visitor.TableCollector;
import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
/** If enable query rewrite with mv, should init materialization context after analyze*/
public class InitMaterializationContextHook implements PlannerHook {
public static final Logger LOG = LogManager.getLogger(InitMaterializationContextHook.class);
public static final InitMaterializationContextHook INSTANCE = new InitMaterializationContextHook();
@Override
public void afterAnalyze(NereidsPlanner planner) {
initMaterializationContext(planner.getCascadesContext());
}
private void initMaterializationContext(CascadesContext cascadesContext) {
Plan rewritePlan = cascadesContext.getRewritePlan();
TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet());
rewritePlan.accept(TableCollector.INSTANCE, collectorContext);
List<TableIf> collectedTables = collectorContext.getCollectedTables();
if (collectedTables.isEmpty()) {
return;
}
List<BaseTableInfo> baseTableUsed =
collectedTables.stream().map(BaseTableInfo::new).collect(Collectors.toList());
// TODO the logic should be move to MTMVCacheManager later when getAvailableMaterializedView is ready in
// MV Cache manager
Env env = cascadesContext.getConnectContext().getEnv();
MTMVCacheManager cacheManager = env.getMtmvService().getCacheManager();
Set<BaseTableInfo> materializedViews = new HashSet<>();
for (BaseTableInfo baseTableInfo : baseTableUsed) {
Set<BaseTableInfo> mtmvsByBaseTable = cacheManager.getMtmvsByBaseTable(baseTableInfo);
if (mtmvsByBaseTable == null || mtmvsByBaseTable.isEmpty()) {
continue;
}
materializedViews.addAll(mtmvsByBaseTable);
}
if (materializedViews.isEmpty()) {
return;
}
materializedViews.forEach(mvBaseTableInfo -> {
try {
MTMV materializedView = (MTMV) Env.getCurrentInternalCatalog()
.getDbOrMetaException(mvBaseTableInfo.getDbId())
.getTableOrMetaException(mvBaseTableInfo.getTableId(), TableType.MATERIALIZED_VIEW);
String qualifiedName = materializedView.getQualifiedName();
// generate outside, maybe add partition filter in the future
Plan mvScan = new LogicalOlapScan(cascadesContext.getStatementContext().getNextRelationId(),
(OlapTable) materializedView,
ImmutableList.of(qualifiedName),
Lists.newArrayList(materializedView.getId()),
Lists.newArrayList(),
Optional.empty());
List<NamedExpression> mvProjects = mvScan.getOutput().stream().map(NamedExpression.class::cast)
.collect(Collectors.toList());
mvScan = new LogicalProject<Plan>(mvProjects, mvScan);
cascadesContext.addMaterializationContext(
MaterializationContext.fromMaterializedView(materializedView, mvScan, cascadesContext));
} catch (MetaNotFoundException metaNotFoundException) {
LOG.error(mvBaseTableInfo.toString() + " can not find corresponding materialized view.");
}
});
}
}

100
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java Normal file
View File

@ -0,0 +1,100 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode;
import org.apache.doris.nereids.rules.exploration.mv.mapping.Mapping.MappedRelation;
import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import java.util.HashMap;
import java.util.Map;
/**
* For outer join we should check the outer join compatibility between query and view
*/
public class LogicalCompatibilityContext {
private BiMap<StructInfoNode, StructInfoNode> queryToViewNodeMapping;
private BiMap<Expression, Expression> queryToViewEdgeExpressionMapping;
public LogicalCompatibilityContext(BiMap<StructInfoNode, StructInfoNode> queryToViewNodeMapping,
BiMap<Expression, Expression> queryToViewEdgeExpressionMapping) {
this.queryToViewNodeMapping = queryToViewNodeMapping;
this.queryToViewEdgeExpressionMapping = queryToViewEdgeExpressionMapping;
}
public BiMap<StructInfoNode, StructInfoNode> getQueryToViewNodeMapping() {
return queryToViewNodeMapping;
}
public BiMap<Expression, Expression> getQueryToViewEdgeExpressionMapping() {
return queryToViewEdgeExpressionMapping;
}
/**
* generate logical compatibility context
*/
public static LogicalCompatibilityContext from(RelationMapping relationMapping,
SlotMapping slotMapping,
StructInfo queryStructInfo,
StructInfo viewStructInfo) {
// init node mapping
BiMap<StructInfoNode, StructInfoNode> queryToViewNodeMapping = HashBiMap.create();
Map<RelationId, StructInfoNode> queryRelationIdStructInfoNodeMap
= queryStructInfo.getRelationIdStructInfoNodeMap();
Map<RelationId, StructInfoNode> viewRelationIdStructInfoNodeMap
= viewStructInfo.getRelationIdStructInfoNodeMap();
for (Map.Entry<MappedRelation, MappedRelation> relationMappingEntry :
relationMapping.getMappedRelationMap().entrySet()) {
StructInfoNode queryStructInfoNode = queryRelationIdStructInfoNodeMap.get(
relationMappingEntry.getKey().getRelationId());
StructInfoNode viewStructInfoNode = viewRelationIdStructInfoNodeMap.get(
relationMappingEntry.getValue().getRelationId());
if (queryStructInfoNode != null && viewStructInfoNode != null) {
queryToViewNodeMapping.put(queryStructInfoNode, viewStructInfoNode);
}
}
// init expression mapping
Map<SlotReference, SlotReference> viewToQuerySlotMapping = slotMapping.inverse().toSlotReferenceMap();
Map<Expression, Expression> queryShuttledExprToExprMap =
queryStructInfo.getShuttledHashConjunctsToConjunctsMap();
Map<Expression, Expression> viewShuttledExprToExprMap =
viewStructInfo.getShuttledHashConjunctsToConjunctsMap();
final Map<Expression, Expression> viewEdgeToConjunctsMapQueryBased = new HashMap<>();
viewShuttledExprToExprMap.forEach((shuttledExpr, expr) -> {
viewEdgeToConjunctsMapQueryBased.put(
ExpressionUtils.replace(shuttledExpr, viewToQuerySlotMapping),
expr);
});
BiMap<Expression, Expression> queryToViewEdgeMapping = HashBiMap.create();
queryShuttledExprToExprMap.forEach((exprSet, edge) -> {
Expression viewExpr = viewEdgeToConjunctsMapQueryBased.get(exprSet);
if (viewExpr != null) {
queryToViewEdgeMapping.put(edge, viewExpr);
}
});
return new LogicalCompatibilityContext(queryToViewNodeMapping, queryToViewEdgeMapping);
}
}

88
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
View File

@ -17,52 +17,110 @@
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.View;
import org.apache.doris.mtmv.MVCache;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.memo.GroupId;
import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ImmutableList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Maintain the context for query rewrite by materialized view
*/
public class MaterializationContext {
// TODO add MaterializedView class
private final Plan mvPlan;
private final CascadesContext context;
private MTMV mtmv;
// Should use stmt id generator in query context
private final Plan mvScanPlan;
private final List<Table> baseTables;
private final List<View> baseViews;
private final List<Table> baseViews;
// Group ids that are rewritten by this mv to reduce rewrite times
private final Set<GroupId> matchedGroups = new HashSet<>();
private final Plan scanPlan;
// generate form mv scan plan
private ExpressionMapping viewExpressionMapping;
public MaterializationContext(Plan mvPlan, CascadesContext context,
List<Table> baseTables, List<View> baseViews, Plan scanPlan) {
this.mvPlan = mvPlan;
this.context = context;
/**
* MaterializationContext, this contains necessary info for query rewriting by mv
*/
public MaterializationContext(MTMV mtmv,
Plan mvScanPlan,
CascadesContext cascadesContext,
List<Table> baseTables,
List<Table> baseViews) {
this.mtmv = mtmv;
this.mvScanPlan = mvScanPlan;
this.baseTables = baseTables;
this.baseViews = baseViews;
this.scanPlan = scanPlan;
MVCache mvCache = mtmv.getMvCache();
// TODO This logic should move to materialized view cache manager
if (mvCache == null) {
mvCache = MVCache.from(mtmv, cascadesContext.getConnectContext());
mtmv.setMvCache(mvCache);
}
List<NamedExpression> mvOutputExpressions = mvCache.getMvOutputExpressions();
// mv output expression shuttle, this will be used to expression rewrite
mvOutputExpressions =
ExpressionUtils.shuttleExpressionWithLineage(mvOutputExpressions, mvCache.getLogicalPlan()).stream()
.map(NamedExpression.class::cast)
.collect(Collectors.toList());
this.viewExpressionMapping = ExpressionMapping.generate(
mvOutputExpressions,
mvScanPlan.getExpressions());
}
public Set<GroupId> getMatchedGroups() {
return matchedGroups;
}
public boolean alreadyRewrite(GroupId groupId) {
return this.matchedGroups.contains(groupId);
}
public void addMatchedGroup(GroupId groupId) {
matchedGroups.add(groupId);
}
public Plan getMvPlan() {
return mvPlan;
public MTMV getMtmv() {
return mtmv;
}
public Plan getScanPlan() {
return scanPlan;
public Plan getMvScanPlan() {
return mvScanPlan;
}
public List<Table> getBaseTables() {
return baseTables;
}
public List<Table> getBaseViews() {
return baseViews;
}
public ExpressionMapping getViewExpressionIndexMapping() {
return viewExpressionMapping;
}
/**
* MaterializationContext fromMaterializedView
*/
public static MaterializationContext fromMaterializedView(MTMV materializedView,
Plan mvScanPlan,
CascadesContext cascadesContext) {
return new MaterializationContext(
materializedView,
mvScanPlan,
cascadesContext,
ImmutableList.of(),
ImmutableList.of());
}
}

81
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java
View File

@ -23,36 +23,48 @@ import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.Optional;
import java.util.stream.Collectors;
/**
* This record the predicates which can be pulled up or some other type predicates
* */
*/
public class Predicates {
// Predicates that can be pulled up
private final Set<Expression> pulledUpPredicates;
private final List<Expression> pulledUpPredicates = new ArrayList<>();
public Predicates(Set<Expression> pulledUpPredicates) {
this.pulledUpPredicates = pulledUpPredicates;
private Predicates() {
}
public static Predicates of(Set<Expression> pulledUpPredicates) {
return new Predicates(pulledUpPredicates);
public static Predicates of() {
return new Predicates();
}
public Set<Expression> getPulledUpPredicates() {
public static Predicates of(List<? extends Expression> pulledUpPredicates) {
Predicates predicates = new Predicates();
pulledUpPredicates.forEach(predicates::addPredicate);
return predicates;
}
public List<? extends Expression> getPulledUpPredicates() {
return pulledUpPredicates;
}
public void addPredicate(Expression expression) {
this.pulledUpPredicates.add(expression);
}
public Expression composedExpression() {
return ExpressionUtils.and(pulledUpPredicates);
return ExpressionUtils.and(pulledUpPredicates.stream().map(Expression.class::cast)
.collect(Collectors.toList()));
}
/**
* Split the expression to equal, range and residual predicate.
* */
*/
public static SplitPredicate splitPredicates(Expression expression) {
PredicatesSplitter predicatesSplit = new PredicatesSplitter(expression);
return predicatesSplit.getSplitPredicate();
@ -60,28 +72,28 @@ public class Predicates {
/**
* The split different representation for predicate expression, such as equal, range and residual predicate.
* */
*/
public static final class SplitPredicate {
private final Expression equalPredicate;
private final Expression rangePredicate;
private final Expression residualPredicate;
private Optional<Expression> equalPredicate;
private Optional<Expression> rangePredicate;
private Optional<Expression> residualPredicate;
public SplitPredicate(Expression equalPredicate, Expression rangePredicate, Expression residualPredicate) {
this.equalPredicate = equalPredicate;
this.rangePredicate = rangePredicate;
this.residualPredicate = residualPredicate;
this.equalPredicate = Optional.ofNullable(equalPredicate);
this.rangePredicate = Optional.ofNullable(rangePredicate);
this.residualPredicate = Optional.ofNullable(residualPredicate);
}
public Expression getEqualPredicate() {
return equalPredicate;
return equalPredicate.orElse(BooleanLiteral.TRUE);
}
public Expression getRangePredicate() {
return rangePredicate;
return rangePredicate.orElse(BooleanLiteral.TRUE);
}
public Expression getResidualPredicate() {
return residualPredicate;
return residualPredicate.orElse(BooleanLiteral.TRUE);
}
public static SplitPredicate empty() {
@ -90,7 +102,7 @@ public class Predicates {
/**
* SplitPredicate construct
* */
*/
public static SplitPredicate of(Expression equalPredicates,
Expression rangePredicates,
Expression residualPredicates) {
@ -99,27 +111,32 @@ public class Predicates {
/**
* isEmpty
* */
*/
public boolean isEmpty() {
return equalPredicate == null
&& rangePredicate == null
&& residualPredicate == null;
return !equalPredicate.isPresent()
&& !rangePredicate.isPresent()
&& !residualPredicate.isPresent();
}
public List<Expression> toList() {
return ImmutableList.of(equalPredicate, rangePredicate, residualPredicate);
return ImmutableList.of(equalPredicate.orElse(BooleanLiteral.TRUE),
rangePredicate.orElse(BooleanLiteral.TRUE),
residualPredicate.orElse(BooleanLiteral.TRUE));
}
/**
* Check the predicates in SplitPredicate is whether all true or not
*/
public boolean isAlwaysTrue() {
return equalPredicate instanceof BooleanLiteral
&& rangePredicate instanceof BooleanLiteral
&& residualPredicate instanceof BooleanLiteral
&& ((BooleanLiteral) equalPredicate).getValue()
&& ((BooleanLiteral) rangePredicate).getValue()
&& ((BooleanLiteral) residualPredicate).getValue();
Expression equalExpr = equalPredicate.orElse(BooleanLiteral.TRUE);
Expression rangeExpr = rangePredicate.orElse(BooleanLiteral.TRUE);
Expression residualExpr = residualPredicate.orElse(BooleanLiteral.TRUE);
return equalExpr instanceof BooleanLiteral
&& rangeExpr instanceof BooleanLiteral
&& residualExpr instanceof BooleanLiteral
&& ((BooleanLiteral) equalExpr).getValue()
&& ((BooleanLiteral) rangeExpr).getValue()
&& ((BooleanLiteral) residualExpr).getValue();
}
}
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java
View File

@ -86,7 +86,7 @@ public class PredicatesSplitter {
residualPredicates.add(compoundPredicate);
return null;
}
return super.visit(compoundPredicate, context);
return super.visitCompoundPredicate(compoundPredicate, context);
}
}

288
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
View File

@ -18,56 +18,169 @@
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
import org.apache.doris.nereids.trees.plans.algebra.Filter;
import org.apache.doris.nereids.trees.plans.algebra.Join;
import org.apache.doris.nereids.trees.plans.algebra.Project;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
/**
* StructInfo
*/
public class StructInfo {
private final List<CatalogRelation> relations;
private final Predicates predicates;
// Used by predicate compensation
private final EquivalenceClass equivalenceClass;
public static final JoinPatternChecker JOIN_PATTERN_CHECKER = new JoinPatternChecker();
// struct info splitter
public static final PlanSplitter PLAN_SPLITTER = new PlanSplitter();
private static final RelationCollector RELATION_COLLECTOR = new RelationCollector();
private static final PredicateCollector PREDICATE_COLLECTOR = new PredicateCollector();
// source data
private final Plan originalPlan;
private final HyperGraph hyperGraph;
private boolean valid = true;
// derived data following
// top plan which may include project or filter, except for join and scan
private Plan topPlan;
// bottom plan which top plan only contain join or scan. this is needed by hyper graph
private Plan bottomPlan;
private final List<CatalogRelation> relations = new ArrayList<>();
// this is for LogicalCompatibilityContext later
private final Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap = new HashMap<>();
private Predicates predicates;
private SplitPredicate splitPredicate;
private EquivalenceClass equivalenceClass;
// this is for LogicalCompatibilityContext later
private final Map<Expression, Expression> shuttledHashConjunctsToConjunctsMap = new HashMap<>();
private StructInfo(List<CatalogRelation> relations,
Predicates predicates,
Plan originalPlan,
HyperGraph hyperGraph) {
this.relations = relations;
this.predicates = predicates;
private StructInfo(Plan originalPlan, @Nullable Plan topPlan, @Nullable Plan bottomPlan, HyperGraph hyperGraph) {
this.originalPlan = originalPlan;
this.hyperGraph = hyperGraph;
this.topPlan = topPlan;
this.bottomPlan = bottomPlan;
init();
}
private void init() {
if (topPlan == null || bottomPlan == null) {
PlanSplitContext planSplitContext = new PlanSplitContext(Sets.newHashSet(LogicalJoin.class));
originalPlan.accept(PLAN_SPLITTER, planSplitContext);
this.bottomPlan = planSplitContext.getBottomPlan();
this.topPlan = planSplitContext.getTopPlan();
}
this.predicates = Predicates.of();
// Collect predicate from join condition in hyper graph
this.hyperGraph.getEdges().forEach(edge -> {
List<Expression> hashJoinConjuncts = edge.getHashJoinConjuncts();
hashJoinConjuncts.forEach(conjunctExpr -> {
predicates.addPredicate(conjunctExpr);
// shuttle expression in edge for LogicalCompatibilityContext later
shuttledHashConjunctsToConjunctsMap.put(
ExpressionUtils.shuttleExpressionWithLineage(
Lists.newArrayList(conjunctExpr), edge.getJoin()).get(0),
conjunctExpr);
});
List<Expression> otherJoinConjuncts = edge.getOtherJoinConjuncts();
if (!otherJoinConjuncts.isEmpty()) {
this.valid = false;
}
});
if (!this.isValid()) {
return;
}
// Collect predicate from filter node in hyper graph
this.hyperGraph.getNodes().forEach(node -> {
// plan relation collector and set to map
Plan nodePlan = node.getPlan();
List<CatalogRelation> nodeRelations = new ArrayList<>();
nodePlan.accept(RELATION_COLLECTOR, nodeRelations);
this.relations.addAll(nodeRelations);
// every node should only have one relation, this is for LogicalCompatibilityContext
relationIdStructInfoNodeMap.put(nodeRelations.get(0).getRelationId(), (StructInfoNode) node);
// if inner join add where condition
Set<Expression> predicates = new HashSet<>();
nodePlan.accept(PREDICATE_COLLECTOR, predicates);
predicates.forEach(this.predicates::addPredicate);
});
// TODO Collect predicate from top plan not in hyper graph, should optimize, twice now
Set<Expression> topPlanPredicates = new HashSet<>();
topPlan.accept(PREDICATE_COLLECTOR, topPlanPredicates);
topPlanPredicates.forEach(this.predicates::addPredicate);
// construct equivalenceClass according to equals predicates
this.equivalenceClass = new EquivalenceClass();
SplitPredicate splitPredicate = Predicates.splitPredicates(predicates.composedExpression());
List<Expression> shuttledExpression = ExpressionUtils.shuttleExpressionWithLineage(
this.predicates.getPulledUpPredicates(), originalPlan).stream()
.map(Expression.class::cast)
.collect(Collectors.toList());
SplitPredicate splitPredicate = Predicates.splitPredicates(ExpressionUtils.and(shuttledExpression));
this.splitPredicate = splitPredicate;
for (Expression expression : ExpressionUtils.extractConjunction(splitPredicate.getEqualPredicate())) {
EqualTo equalTo = (EqualTo) expression;
equivalenceClass.addEquivalenceClass(
(SlotReference) equalTo.getArguments().get(0),
(SlotReference) equalTo.getArguments().get(1));
if (expression instanceof BooleanLiteral && ((BooleanLiteral) expression).getValue()) {
continue;
}
if (expression instanceof EqualTo) {
EqualTo equalTo = (EqualTo) expression;
equivalenceClass.addEquivalenceClass(
(SlotReference) equalTo.getArguments().get(0),
(SlotReference) equalTo.getArguments().get(1));
}
}
}
public static StructInfo of(Plan originalPlan) {
// TODO build graph from original plan and get relations and predicates from graph
return new StructInfo(null, null, originalPlan, null);
/**
* Build Struct info from plan.
* Maybe return multi structInfo when original plan already be rewritten by mv
*/
public static List<StructInfo> of(Plan originalPlan) {
// TODO only consider the inner join currently, Should support outer join
// Split plan by the boundary which contains multi child
PlanSplitContext planSplitContext = new PlanSplitContext(Sets.newHashSet(LogicalJoin.class));
originalPlan.accept(PLAN_SPLITTER, planSplitContext);
List<HyperGraph> structInfos = HyperGraph.toStructInfo(planSplitContext.getBottomPlan());
return structInfos.stream()
.map(hyperGraph -> new StructInfo(originalPlan, planSplitContext.getTopPlan(),
planSplitContext.getBottomPlan(), hyperGraph))
.collect(Collectors.toList());
}
/**
* Build Struct info from group.
* Maybe return multi structInfo when original plan already be rewritten by mv
*/
public static StructInfo of(Group group) {
// TODO build graph from original plan and get relations and predicates from graph
return new StructInfo(null, null, group.getLogicalExpression().getPlan(), null);
return null;
}
public List<CatalogRelation> getRelations() {
@ -90,6 +203,30 @@ public class StructInfo {
return hyperGraph;
}
public SplitPredicate getSplitPredicate() {
return splitPredicate;
}
public boolean isValid() {
return valid;
}
public Plan getTopPlan() {
return topPlan;
}
public Plan getBottomPlan() {
return bottomPlan;
}
public Map<RelationId, StructInfoNode> getRelationIdStructInfoNodeMap() {
return relationIdStructInfoNodeMap;
}
public Map<Expression, Expression> getShuttledHashConjunctsToConjunctsMap() {
return shuttledHashConjunctsToConjunctsMap;
}
public List<? extends Expression> getExpressions() {
return originalPlan instanceof LogicalProject
? ((LogicalProject<Plan>) originalPlan).getProjects() : originalPlan.getOutput();
@ -99,8 +236,115 @@ public class StructInfo {
* Judge the source graph logical is whether the same as target
* For inner join should judge only the join tables,
* for other join type should also judge the join direction, it's input filter that can not be pulled up etc.
* */
public static boolean isGraphLogicalEquals(HyperGraph source, HyperGraph target) {
return false;
*/
public static boolean isGraphLogicalEquals(StructInfo queryStructInfo, StructInfo viewStructInfo,
LogicalCompatibilityContext compatibilityContext) {
// TODO: if not inner join, should check the join graph logical equivalence
return true;
}
private static class RelationCollector extends DefaultPlanVisitor<Void, List<CatalogRelation>> {
@Override
public Void visit(Plan plan, List<CatalogRelation> collectedRelations) {
if (plan instanceof CatalogRelation) {
collectedRelations.add((CatalogRelation) plan);
}
return super.visit(plan, collectedRelations);
}
}
private static class PredicateCollector extends DefaultPlanVisitor<Void, Set<Expression>> {
@Override
public Void visit(Plan plan, Set<Expression> predicates) {
if (plan instanceof LogicalFilter) {
predicates.add(((LogicalFilter) plan).getPredicate());
}
return super.visit(plan, predicates);
}
}
/**
* Split the plan into bottom and up, the boundary is given by context,
* the bottom contains the boundary.
*/
public static class PlanSplitter extends DefaultPlanVisitor<Void, PlanSplitContext> {
@Override
public Void visit(Plan plan, PlanSplitContext context) {
if (context.getTopPlan() == null) {
context.setTopPlan(plan);
}
if (context.isBoundary(plan)) {
context.setBottomPlan(plan);
return null;
}
return super.visit(plan, context);
}
}
/**
* Plan split context, this hold bottom and top plan, and boundary plan setting
*/
public static class PlanSplitContext {
private Plan bottomPlan;
private Plan topPlan;
private Set<Class<? extends Plan>> boundaryPlanClazzSet;
public PlanSplitContext(Set<Class<? extends Plan>> boundaryPlanClazzSet) {
this.boundaryPlanClazzSet = boundaryPlanClazzSet;
}
public Plan getBottomPlan() {
return bottomPlan;
}
public void setBottomPlan(Plan bottomPlan) {
this.bottomPlan = bottomPlan;
}
public Plan getTopPlan() {
return topPlan;
}
public void setTopPlan(Plan topPlan) {
this.topPlan = topPlan;
}
/**
* isBoundary
*/
public boolean isBoundary(Plan plan) {
for (Class<? extends Plan> boundaryPlanClazz : boundaryPlanClazzSet) {
if (boundaryPlanClazz.isAssignableFrom(plan.getClass())) {
return true;
}
}
return false;
}
}
/**
* JoinPatternChecker
*/
public static class JoinPatternChecker extends DefaultPlanVisitor<Boolean, Set<JoinType>> {
@Override
public Boolean visit(Plan plan, Set<JoinType> requiredJoinType) {
super.visit(plan, requiredJoinType);
if (!(plan instanceof Filter)
&& !(plan instanceof Project)
&& !(plan instanceof CatalogRelation)
&& !(plan instanceof Join)) {
return false;
}
if (plan instanceof Join) {
Join join = (Join) plan;
if (!requiredJoinType.contains(join.getJoinType())) {
return false;
}
if (!join.getOtherJoinConjuncts().isEmpty()) {
return false;
}
}
return true;
}
}
}

68
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/EquivalenceClassSetMapping.java Normal file
View File

@ -0,0 +1,68 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.exploration.mv.mapping;
import org.apache.doris.nereids.rules.exploration.mv.EquivalenceClass;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* EquivalenceClassSetMapping
* This will extract the equivalence class set in EquivalenceClass and mapping set in
* two different EquivalenceClass.
*/
public class EquivalenceClassSetMapping extends Mapping {
private final Map<Set<SlotReference>, Set<SlotReference>> equivalenceClassSetMap;
public EquivalenceClassSetMapping(Map<Set<SlotReference>,
Set<SlotReference>> equivalenceClassSetMap) {
this.equivalenceClassSetMap = equivalenceClassSetMap;
}
public static EquivalenceClassSetMapping of(Map<Set<SlotReference>, Set<SlotReference>> equivalenceClassSetMap) {
return new EquivalenceClassSetMapping(equivalenceClassSetMap);
}
/**
* Generate source equivalence set map to target equivalence set
*/
public static EquivalenceClassSetMapping generate(EquivalenceClass source, EquivalenceClass target) {
Map<Set<SlotReference>, Set<SlotReference>> equivalenceClassSetMap = new HashMap<>();
List<Set<SlotReference>> sourceSets = source.getEquivalenceSetList();
List<Set<SlotReference>> targetSets = target.getEquivalenceSetList();
for (Set<SlotReference> sourceSet : sourceSets) {
for (Set<SlotReference> targetSet : targetSets) {
if (sourceSet.containsAll(targetSet)) {
equivalenceClassSetMap.put(sourceSet, targetSet);
}
}
}
return EquivalenceClassSetMapping.of(equivalenceClassSetMap);
}
public Map<Set<SlotReference>, Set<SlotReference>> getEquivalenceClassSetMap() {
return equivalenceClassSetMap;
}
}

100
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java Normal file
View File

@ -0,0 +1,100 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.exploration.mv.mapping;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Expression mapping, maybe one expression map to multi expression
*/
public class ExpressionMapping extends Mapping {
private final Multimap<? extends Expression, ? extends Expression> expressionMapping;
public ExpressionMapping(Multimap<? extends Expression, ? extends Expression> expressionMapping) {
this.expressionMapping = expressionMapping;
}
public Multimap<? extends Expression, ? extends Expression> getExpressionMapping() {
return expressionMapping;
}
/**
* ExpressionMapping flatten
*/
public List<Map<? extends Expression, ? extends Expression>> flattenMap() {
List<List<Pair<Expression, Expression>>> tmpExpressionPairs = new ArrayList<>(this.expressionMapping.size());
Map<? extends Expression, ? extends Collection<? extends Expression>> expressionMappingMap =
expressionMapping.asMap();
for (Map.Entry<? extends Expression, ? extends Collection<? extends Expression>> entry
: expressionMappingMap.entrySet()) {
List<Pair<Expression, Expression>> targetExpressionList = new ArrayList<>(entry.getValue().size());
for (Expression valueExpression : entry.getValue()) {
targetExpressionList.add(Pair.of(entry.getKey(), valueExpression));
}
tmpExpressionPairs.add(targetExpressionList);
}
List<List<Pair<Expression, Expression>>> cartesianExpressionMap = Lists.cartesianProduct(tmpExpressionPairs);
final List<Map<? extends Expression, ? extends Expression>> flattenedMap = new ArrayList<>();
for (List<Pair<Expression, Expression>> listPair : cartesianExpressionMap) {
final Map<Expression, Expression> expressionMap = new HashMap<>();
listPair.forEach(pair -> expressionMap.put(pair.key(), pair.value()));
flattenedMap.add(expressionMap);
}
return flattenedMap;
}
/**Permute the key of expression mapping. this is useful for expression rewrite, if permute key to query based
* then when expression rewrite success, we can get the mv scan expression directly.
*/
public ExpressionMapping keyPermute(SlotMapping slotMapping) {
Multimap<Expression, Expression> permutedExpressionMapping = ArrayListMultimap.create();
Map<? extends Expression, ? extends Collection<? extends Expression>> expressionMap =
this.getExpressionMapping().asMap();
for (Map.Entry<? extends Expression, ? extends Collection<? extends Expression>> entry :
expressionMap.entrySet()) {
Expression replacedExpr = ExpressionUtils.replace(entry.getKey(), slotMapping.toSlotReferenceMap());
permutedExpressionMapping.putAll(replacedExpr, entry.getValue());
}
return new ExpressionMapping(permutedExpressionMapping);
}
/**ExpressionMapping generate*/
public static ExpressionMapping generate(
List<? extends Expression> sourceExpressions,
List<? extends Expression> targetExpressions) {
final Multimap<Expression, Expression> expressionMultiMap =
ArrayListMultimap.create();
for (int i = 0; i < sourceExpressions.size(); i++) {
expressionMultiMap.put(sourceExpressions.get(i), targetExpressions.get(i));
}
return new ExpressionMapping(expressionMultiMap);
}
}

36
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java
View File

@ -17,12 +17,13 @@
package org.apache.doris.nereids.rules.exploration.mv.mapping;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import java.util.HashMap;
import java.util.Map;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
@ -32,19 +33,19 @@ import javax.annotation.Nullable;
*/
public class SlotMapping extends Mapping {
private final BiMap<MappedSlot, MappedSlot> slotMapping;
private final BiMap<MappedSlot, MappedSlot> relationSlotMap;
private Map<SlotReference, SlotReference> slotReferenceMap;
public SlotMapping(BiMap<MappedSlot, MappedSlot> slotMapping) {
this.slotMapping = slotMapping;
public SlotMapping(BiMap<MappedSlot, MappedSlot> relationSlotMap) {
this.relationSlotMap = relationSlotMap;
}
public BiMap<MappedSlot, MappedSlot> getSlotBiMap() {
return slotMapping;
public BiMap<MappedSlot, MappedSlot> getRelationSlotMap() {
return relationSlotMap;
}
public SlotMapping inverse() {
return slotMapping == null
? SlotMapping.of(HashBiMap.create()) : SlotMapping.of(slotMapping.inverse());
return SlotMapping.of(relationSlotMap.inverse());
}
public static SlotMapping of(BiMap<MappedSlot, MappedSlot> relationSlotMap) {
@ -75,10 +76,23 @@ public class SlotMapping extends Mapping {
return SlotMapping.of(relationSlotMap);
}
public Map<MappedSlot, MappedSlot> toMappedSlotMap() {
return (Map) this.getRelationSlotMap();
}
/**
* SlotMapping, getSlotMap
* SlotMapping, toSlotReferenceMap
*/
public Map<? extends Expression, ? extends Expression> getSlotMap() {
return (Map) this.getSlotBiMap();
public Map<SlotReference, SlotReference> toSlotReferenceMap() {
if (this.slotReferenceMap != null) {
return this.slotReferenceMap;
}
Map<SlotReference, SlotReference> slotReferenceSlotReferenceMap = new HashMap<>();
for (Map.Entry<MappedSlot, MappedSlot> entry : this.getRelationSlotMap().entrySet()) {
slotReferenceSlotReferenceMap.put((SlotReference) entry.getKey().getSlot(),
(SlotReference) entry.getValue().getSlot());
}
this.slotReferenceMap = slotReferenceSlotReferenceMap;
return this.slotReferenceMap;
}
}

14
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java
View File

@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableSet;
import java.util.ArrayDeque;
import java.util.Deque;
import java.util.List;
import java.util.Set;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Function;
@ -210,6 +211,19 @@ public interface TreeNode<NODE_TYPE extends TreeNode<NODE_TYPE>> {
return (List<T>) result.build();
}
/**
* Collect the nodes that satisfied the predicate to set.
*/
default <T> Set<T> collectToSet(Predicate<TreeNode<NODE_TYPE>> predicate) {
ImmutableSet.Builder<TreeNode<NODE_TYPE>> result = ImmutableSet.builder();
foreach(node -> {
if (predicate.test(node)) {
result.add(node);
}
});
return (Set<T>) result.build();
}
/**
* iterate top down and test predicate if contains any instance of the classes
* @param types classes array

10
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ExprId.java
View File

@ -45,4 +45,14 @@ public class ExprId extends Id<ExprId> {
public String toString() {
return "" + id;
}
@Override
public boolean equals(Object obj) {
return super.equals(obj);
}
@Override
public int hashCode() {
return super.hashCode();
}
}

10
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/RelationId.java
View File

@ -46,4 +46,14 @@ public class RelationId extends Id<RelationId> {
public String toString() {
return "RelationId#" + id;
}
@Override
public boolean equals(Object obj) {
return super.equals(obj);
}
@Override
public int hashCode() {
return super.hashCode();
}
}

4
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainCommand.java
View File

@ -21,6 +21,7 @@ import org.apache.doris.analysis.ExplainOptions;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook;
import org.apache.doris.nereids.trees.plans.Explainable;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
@ -78,6 +79,9 @@ public class ExplainCommand extends Command implements NoForward {
logicalPlanAdapter.setIsExplain(new ExplainOptions(level));
executor.setParsedStmt(logicalPlanAdapter);
NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext());
if (ctx.getSessionVariable().isEnableMaterializedViewRewrite()) {
planner.addHook(InitMaterializationContextHook.INSTANCE);
}
planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift());
executor.setPlanner(planner);
executor.checkBlockRules();

2
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java
View File

@ -39,7 +39,7 @@ public class TableCollector extends DefaultPlanVisitor<Void, TableCollectorConte
public Void visit(Plan plan, TableCollectorContext context) {
if (plan instanceof CatalogRelation) {
TableIf table = ((CatalogRelation) plan).getTable();
if (context.getTargetTableTypes().contains(table.getType())) {
if (context.getTargetTableTypes().isEmpty() || context.getTargetTableTypes().contains(table.getType())) {
context.getCollectedTables().add(table);
}
}

84
fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
View File

@ -18,6 +18,8 @@
package org.apache.doris.nereids.util;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.MaterializedViewException;
import org.apache.doris.common.NereidsException;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
import org.apache.doris.nereids.rules.expression.rules.FoldConstantRule;
@ -205,6 +207,11 @@ public class ExpressionUtils {
.orElse(BooleanLiteral.of(type == And.class));
}
public static List<? extends Expression> shuttleExpressionWithLineage(List<? extends Expression> expressions,
Plan plan) {
return shuttleExpressionWithLineage(expressions, plan, ImmutableSet.of(), ImmutableSet.of());
}
/**
* Replace the slot in expressions with the lineage identifier from specifiedbaseTable sets or target table types
* example as following:
@ -221,13 +228,18 @@ public class ExpressionUtils {
ExpressionLineageReplacer.ExpressionReplaceContext replaceContext =
new ExpressionLineageReplacer.ExpressionReplaceContext(
expressions.stream().map(NamedExpression.class::cast).collect(Collectors.toList()),
expressions.stream().map(Expression.class::cast).collect(Collectors.toList()),
targetTypes,
tableIdentifiers);
plan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext);
// Replace expressions by expression map
return replaceContext.getReplacedExpressions();
List<Expression> replacedExpressions = replaceContext.getReplacedExpressions();
if (expressions.size() != replacedExpressions.size()) {
throw new NereidsException("shuttle expression fail",
new MaterializedViewException("shuttle expression fail"));
}
return replacedExpressions;
}
/**
@ -298,7 +310,24 @@ public class ExpressionUtils {
* </pre>
*/
public static Expression replace(Expression expr, Map<? extends Expression, ? extends Expression> replaceMap) {
return expr.accept(ExpressionReplacer.INSTANCE, replaceMap);
return expr.accept(ExpressionReplacer.INSTANCE, ExpressionReplacerContext.of(replaceMap, false));
}
/**
* Replace expression node in the expression tree by `replaceMap` in top-down manner.
* if replaced, create alias
* For example.
* <pre>
* input expression: a > 1
* replaceMap: a -> b + c
*
* output:
* ((b + c) as a) > 1
* </pre>
*/
public static Expression replace(Expression expr, Map<? extends Expression, ? extends Expression> replaceMap,
boolean withAlias) {
return expr.accept(ExpressionReplacer.INSTANCE, ExpressionReplacerContext.of(replaceMap, true));
}
/**
@ -306,7 +335,8 @@ public class ExpressionUtils {
*/
public static NamedExpression replace(NamedExpression expr,
Map<? extends Expression, ? extends Expression> replaceMap) {
Expression newExpr = expr.accept(ExpressionReplacer.INSTANCE, replaceMap);
Expression newExpr = expr.accept(ExpressionReplacer.INSTANCE,
ExpressionReplacerContext.of(replaceMap, false));
if (newExpr instanceof NamedExpression) {
return (NamedExpression) newExpr;
} else {
@ -336,18 +366,54 @@ public class ExpressionUtils {
}
private static class ExpressionReplacer
extends DefaultExpressionRewriter<Map<? extends Expression, ? extends Expression>> {
extends DefaultExpressionRewriter<ExpressionReplacerContext> {
public static final ExpressionReplacer INSTANCE = new ExpressionReplacer();
private ExpressionReplacer() {
}
@Override
public Expression visit(Expression expr, Map<? extends Expression, ? extends Expression> replaceMap) {
if (replaceMap.containsKey(expr)) {
return replaceMap.get(expr);
public Expression visit(Expression expr, ExpressionReplacerContext replacerContext) {
Map<? extends Expression, ? extends Expression> replaceMap = replacerContext.getReplaceMap();
boolean isContained = replaceMap.containsKey(expr);
if (!isContained) {
return super.visit(expr, replacerContext);
}
return super.visit(expr, replaceMap);
boolean withAlias = replacerContext.isWithAlias();
if (!withAlias) {
return replaceMap.get(expr);
} else {
Expression replacedExpression = replaceMap.get(expr);
if (replacedExpression instanceof SlotReference) {
replacedExpression = ((SlotReference) (replacedExpression)).withNullable(expr.nullable());
}
return new Alias(((NamedExpression) expr).getExprId(), replacedExpression,
((NamedExpression) expr).getName());
}
}
}
private static class ExpressionReplacerContext {
private final Map<? extends Expression, ? extends Expression> replaceMap;
private final boolean withAlias;
private ExpressionReplacerContext(Map<? extends Expression, ? extends Expression> replaceMap,
boolean withAlias) {
this.replaceMap = replaceMap;
this.withAlias = withAlias;
}
public static ExpressionReplacerContext of(Map<? extends Expression, ? extends Expression> replaceMap,
boolean withAlias) {
return new ExpressionReplacerContext(replaceMap, withAlias);
}
public Map<? extends Expression, ? extends Expression> getReplaceMap() {
return replaceMap;
}
public boolean isWithAlias() {
return withAlias;
}
}

4
fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
View File

@ -42,6 +42,7 @@ import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
import org.apache.doris.nereids.PlannerHook;
import org.apache.doris.qe.CommonResultSet;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ResultSet;
@ -679,4 +680,7 @@ public class OriginalPlanner extends Planner {
ResultSet resultSet = new CommonResultSet(metadata, Collections.singletonList(data));
return Optional.of(resultSet);
}
@Override
public void addHook(PlannerHook hook) {}
}

3
fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java
View File

@ -23,6 +23,7 @@ import org.apache.doris.analysis.StatementBase;
import org.apache.doris.common.UserException;
import org.apache.doris.common.profile.PlanTreeBuilder;
import org.apache.doris.common.profile.PlanTreePrinter;
import org.apache.doris.nereids.PlannerHook;
import org.apache.doris.qe.ResultSet;
import org.apache.doris.thrift.TQueryOptions;
@ -123,4 +124,6 @@ public abstract class Planner {
public abstract Optional<ResultSet> handleQueryInFe(StatementBase parsedStmt);
public abstract void addHook(PlannerHook hook);
}

4
fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
View File

@ -121,6 +121,7 @@ import org.apache.doris.nereids.exceptions.ParseException;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.minidump.MinidumpUtils;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook;
import org.apache.doris.nereids.stats.StatsErrorEstimator;
import org.apache.doris.nereids.trees.plans.commands.BatchInsertIntoTableCommand;
import org.apache.doris.nereids.trees.plans.commands.Command;
@ -576,6 +577,9 @@ public class StmtExecutor {
}
// create plan
planner = new NereidsPlanner(statementContext);
if (context.getSessionVariable().isEnableMaterializedViewRewrite()) {
planner.addHook(InitMaterializationContextHook.INSTANCE);
}
try {
planner.plan(parsedStmt, context.getSessionVariable().toThrift());
checkBlockRules();