database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](mtmv) ensure rewritten plan output order correct even project been eliminated (#31870)

Browse Source
This commit is contained in:
morrySnow
2024-03-08 10:54:57 +08:00
committed by yiguolei
parent 1721bfb87a
commit 679cd0ab45
4 changed files with 43 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
View File

@ -22,7 +22,6 @@ import org.apache.doris.nereids.NereidsPlanner;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
@ -30,9 +29,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.OriginStatement;
import java.util.List;
import java.util.stream.Collectors;
/**
* The cache for materialized view cache
*/
@ -40,20 +36,20 @@ public class MTMVCache {
// the materialized view plan which should be optimized by the same rules to query
private final Plan logicalPlan;
// this should be shuttle expression with lineage
private final List<NamedExpression> mvOutputExpressions;
// for stable output order, we should use original plan
private final Plan originalPlan;
public MTMVCache(Plan logicalPlan, List<NamedExpression> mvOutputExpressions) {
public MTMVCache(Plan logicalPlan, Plan originalPlan) {
this.logicalPlan = logicalPlan;
this.mvOutputExpressions = mvOutputExpressions;
this.originalPlan = originalPlan;
}
public Plan getLogicalPlan() {
return logicalPlan;
}
public List<NamedExpression> getMvOutputExpressions() {
return mvOutputExpressions;
public Plan getOriginalPlan() {
return originalPlan;
}
public static MTMVCache from(MTMV mtmv, ConnectContext connectContext) {
@ -66,15 +62,10 @@ public class MTMVCache {
if (mvSqlStatementContext.getConnectContext().getStatementContext() == null) {
mvSqlStatementContext.getConnectContext().setStatementContext(mvSqlStatementContext);
}
Plan mvRewrittenPlan =
planner.plan(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN);
Plan mvRewrittenPlan = planner.plan(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN);
// TODO: should use visitor or a new rule to remove result sink node
Plan mvPlan = mvRewrittenPlan instanceof LogicalResultSink
? (Plan) ((LogicalResultSink) mvRewrittenPlan).child() : mvRewrittenPlan;
// use rewritten plan output expression currently, if expression rewrite fail,
// consider to use the analyzed plan for output expressions only
List<NamedExpression> mvOutputExpressions = mvPlan.getOutput().stream()
.map(NamedExpression.class::cast)
.collect(Collectors.toList());
return new MTMVCache(mvPlan, mvOutputExpressions);
return new MTMVCache(mvPlan, mvRewrittenPlan);
}
}

52
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View File

@ -35,6 +35,7 @@ import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Not;
@ -55,6 +56,7 @@ import org.apache.doris.nereids.util.TypeUtils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import java.util.ArrayList;
@ -259,27 +261,32 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
* Rewrite by rules and try to make output is the same after optimize by rules
*/
protected Plan rewriteByRules(CascadesContext cascadesContext, Plan rewrittenPlan, Plan originPlan) {
List<Slot> originOutputs = originPlan.getOutput();
if (originOutputs.size() != rewrittenPlan.getOutput().size()) {
return null;
}
Map<Slot, ExprId> originSlotToRewrittenExprId = Maps.newHashMap();
for (int i = 0; i < originOutputs.size(); i++) {
originSlotToRewrittenExprId.put(originOutputs.get(i), rewrittenPlan.getOutput().get(i).getExprId());
}
// run rbo job on mv rewritten plan
CascadesContext rewrittenPlanContext = CascadesContext.initContext(
cascadesContext.getStatementContext(), rewrittenPlan,
cascadesContext.getCurrentJobContext().getRequiredProperties());
Rewriter.getWholeTreeRewriter(rewrittenPlanContext).execute();
rewrittenPlan = rewrittenPlanContext.getRewritePlan();
List<Slot> originPlanOutput = originPlan.getOutput();
List<Slot> rewrittenPlanOutput = rewrittenPlan.getOutput();
if (originPlanOutput.size() != rewrittenPlanOutput.size()) {
return null;
// for get right nullable after rewritten, we need this map
Map<ExprId, Slot> exprIdToNewRewrittenSlot = Maps.newHashMap();
for (Slot slot : rewrittenPlan.getOutput()) {
exprIdToNewRewrittenSlot.put(slot.getExprId(), slot);
}
List<NamedExpression> expressions = new ArrayList<>();
// should add project above rewritten plan if top plan is not project, if aggregate above will nu
if (!isOutputValid(originPlan, rewrittenPlan)) {
for (int i = 0; i < originPlanOutput.size(); i++) {
expressions.add(((NamedExpression) normalizeExpression(originPlanOutput.get(i),
rewrittenPlanOutput.get(i))));
}
return new LogicalProject<>(expressions, rewrittenPlan, false);
}
return rewrittenPlan;
// normalize nullable
ImmutableList<NamedExpression> convertNullable = originOutputs.stream()
.map(s -> normalizeExpression(s, exprIdToNewRewrittenSlot.get(originSlotToRewrittenExprId.get(s))))
.collect(ImmutableList.toImmutableList());
return new LogicalProject<>(convertNullable, rewrittenPlan);
}
/**
@ -398,22 +405,17 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
* Normalize expression with query, keep the consistency of exprId and nullable props with
* query
*/
protected Expression normalizeExpression(Expression sourceExpression, Expression replacedExpression) {
if (sourceExpression instanceof NamedExpression
&& replacedExpression.nullable() != sourceExpression.nullable()) {
private NamedExpression normalizeExpression(
NamedExpression sourceExpression, NamedExpression replacedExpression) {
Expression innerExpression = replacedExpression;
if (replacedExpression.nullable() != sourceExpression.nullable()) {
// if enable join eliminate, query maybe inner join and mv maybe outer join.
// If the slot is at null generate side, the nullable maybe different between query and view
// So need to force to consistent.
replacedExpression = sourceExpression.nullable()
innerExpression = sourceExpression.nullable()
? new Nullable(replacedExpression) : new NonNullable(replacedExpression);
}
if (sourceExpression instanceof NamedExpression
&& !sourceExpression.equals(replacedExpression)) {
NamedExpression sourceNamedExpression = (NamedExpression) sourceExpression;
replacedExpression = new Alias(sourceNamedExpression.getExprId(), replacedExpression,
sourceNamedExpression.getName());
}
return replacedExpression;
return new Alias(sourceExpression.getExprId(), innerExpression, sourceExpression.getName());
}
/**

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
View File

@ -95,7 +95,7 @@ public class InitMaterializationContextHook implements PlannerHook {
// todo should force keep consistency to mv sql plan output
Plan projectScan = new LogicalProject<Plan>(mvProjects, mvScan);
cascadesContext.addMaterializationContext(
MaterializationContext.fromMaterializedView(materializedView, projectScan, cascadesContext));
MaterializationContext.fromMaterializedView(materializedView, projectScan));
});
}
}

19
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
View File

@ -22,7 +22,6 @@ import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Pair;
import org.apache.doris.mtmv.MTMVCache;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.memo.GroupId;
import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
import org.apache.doris.nereids.trees.plans.ObjectId;
@ -47,7 +46,8 @@ import java.util.stream.Collectors;
public class MaterializationContext {
private static final Logger LOG = LogManager.getLogger(MaterializationContext.class);
private MTMV mtmv;
private final MTMV mtmv;
// Should use stmt id generator in query context
private final Plan mvScanPlan;
private final List<Table> baseTables;
@ -68,11 +68,7 @@ public class MaterializationContext {
/**
* MaterializationContext, this contains necessary info for query rewriting by mv
*/
public MaterializationContext(MTMV mtmv,
Plan mvScanPlan,
CascadesContext cascadesContext,
List<Table> baseTables,
List<Table> baseViews) {
public MaterializationContext(MTMV mtmv, Plan mvScanPlan, List<Table> baseTables, List<Table> baseViews) {
this.mtmv = mtmv;
this.mvScanPlan = mvScanPlan;
this.baseTables = baseTables;
@ -91,8 +87,8 @@ public class MaterializationContext {
// mv output expression shuttle, this will be used to expression rewrite
this.mvExprToMvScanExprMapping = ExpressionMapping.generate(
ExpressionUtils.shuttleExpressionWithLineage(
mtmvCache.getMvOutputExpressions(),
mtmvCache.getLogicalPlan()),
mtmvCache.getOriginalPlan().getOutput(),
mtmvCache.getOriginalPlan()),
mvScanPlan.getExpressions());
// copy the plan from cache, which the plan in cache may change
this.mvPlan = mtmvCache.getLogicalPlan();
@ -229,13 +225,10 @@ public class MaterializationContext {
/**
* MaterializationContext fromMaterializedView
*/
public static MaterializationContext fromMaterializedView(MTMV materializedView,
Plan mvScanPlan,
CascadesContext cascadesContext) {
public static MaterializationContext fromMaterializedView(MTMV materializedView, Plan mvScanPlan) {
return new MaterializationContext(
materializedView,
mvScanPlan,
cascadesContext,
ImmutableList.of(),
ImmutableList.of());
}