database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[improvement](mtmv) improve mv rewrite performance by reuse the shuttled expression (#37197) (#37935)

Browse Source 
## Proposed changes

chrry-pick 2.1
pr: https://github.com/apache/doris/pull/37197
commitId: 701c7db4
This commit is contained in:
seawinde
2024-07-17 00:52:58 +08:00
committed by GitHub
parent 359e50fc58
commit dbd2b0abf7
9 changed files with 88 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
View File

@ -108,7 +108,7 @@ public class MTMVCache {
return childContext.getRewritePlan();
}, mvPlan, originPlan);
// Construct structInfo once for use later
Optional<StructInfo> structInfoOptional = MaterializationContext.constructStructInfo(mvPlan,
Optional<StructInfo> structInfoOptional = MaterializationContext.constructStructInfo(mvPlan, originPlan,
planner.getCascadesContext(),
new BitSet());
return new MTMVCache(mvPlan, originPlan, planner.getCascadesContext().getMemo().getRoot().getStatistics(),

4
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View File

@ -145,8 +145,8 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
BitSet materializedViewTableSet) {
List<StructInfo> validStructInfos = new ArrayList<>();
// For every materialized view we should trigger refreshing struct info map
List<StructInfo> uncheckedStructInfos = MaterializedViewUtils.extractStructInfo(queryPlan, cascadesContext,
materializedViewTableSet);
List<StructInfo> uncheckedStructInfos = MaterializedViewUtils.extractStructInfo(queryPlan, queryPlan,
cascadesContext, materializedViewTableSet);
uncheckedStructInfos.forEach(queryStructInfo -> {
boolean valid = checkQueryPattern(queryStructInfo, cascadesContext) && queryStructInfo.isValid();
if (!valid) {

6
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java
View File

@ -50,6 +50,7 @@ public class AsyncMaterializationContext extends MaterializationContext {
private static final Logger LOG = LogManager.getLogger(AsyncMaterializationContext.class);
private final MTMV mtmv;
private List<String> materializationQualifier;
/**
* MaterializationContext, this contains necessary info for query rewriting by mv
@ -72,7 +73,10 @@ public class AsyncMaterializationContext extends MaterializationContext {
@Override
List<String> getMaterializationQualifier() {
return this.mtmv.getFullQualifiers();
if (this.materializationQualifier == null) {
this.materializationQualifier = this.mtmv.getFullQualifiers();
}
return this.materializationQualifier;
}
@Override

34
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
View File

@ -36,7 +36,6 @@ import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.Statistics;
@ -119,32 +118,31 @@ public abstract class MaterializationContext {
this.exprToScanExprMapping.put(originalPlanOutput.get(slotIndex), scanPlanOutput.get(slotIndex));
}
}
this.planOutputShuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage(originalPlanOutput,
originalPlan, new BitSet());
// materialization output expression shuttle, this will be used to expression rewrite
this.shuttledExprToScanExprMapping = ExpressionMapping.generate(
this.planOutputShuttledExpressions,
scanPlanOutput);
// Construct materialization struct info, catch exception which may cause planner roll back
if (structInfo == null) {
Optional<StructInfo> structInfoOptional = constructStructInfo(plan, cascadesContext, new BitSet());
if (!structInfoOptional.isPresent()) {
this.available = false;
}
this.structInfo = structInfoOptional.orElseGet(() -> null);
} else {
this.structInfo = structInfo;
this.structInfo = structInfo == null
? constructStructInfo(plan, originalPlan, cascadesContext, new BitSet()).orElseGet(() -> null)
: structInfo;
this.available = this.structInfo != null;
if (available) {
this.planOutputShuttledExpressions = this.structInfo.getPlanOutputShuttledExpressions();
// materialization output expression shuttle, this will be used to expression rewrite
this.shuttledExprToScanExprMapping = ExpressionMapping.generate(
this.planOutputShuttledExpressions,
scanPlanOutput);
}
}
/**
* Construct materialized view Struct info
* @param plan maybe remove unnecessary plan node, and the logical output maybe wrong
* @param originalPlan original plan, the output is right
*/
public static Optional<StructInfo> constructStructInfo(Plan plan, CascadesContext cascadesContext,
BitSet expectedTableBitSet) {
public static Optional<StructInfo> constructStructInfo(Plan plan, Plan originalPlan,
CascadesContext cascadesContext, BitSet expectedTableBitSet) {
List<StructInfo> viewStructInfos;
try {
viewStructInfos = MaterializedViewUtils.extractStructInfo(plan, cascadesContext, expectedTableBitSet);
viewStructInfos = MaterializedViewUtils.extractStructInfo(plan, originalPlan,
cascadesContext, expectedTableBitSet);
if (viewStructInfos.size() > 1) {
// view struct info should only have one, log error and use the first struct info
LOG.warn(String.format("view strut info is more than one, materialization plan is %s",

8
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
View File

@ -167,8 +167,10 @@ public class MaterializedViewUtils {
/**
* Extract struct info from plan, support to get struct info from logical plan or plan in group.
* @param plan maybe remove unnecessary plan node, and the logical output maybe wrong
* @param originalPlan original plan, the output is right
*/
public static List<StructInfo> extractStructInfo(Plan plan, CascadesContext cascadesContext,
public static List<StructInfo> extractStructInfo(Plan plan, Plan originalPlan, CascadesContext cascadesContext,
BitSet materializedViewTableSet) {
// If plan belong to some group, construct it with group struct info
if (plan.getGroupExpression().isPresent()) {
@ -188,7 +190,7 @@ public class MaterializedViewUtils {
continue;
}
StructInfo structInfo = structInfoMap.getStructInfo(cascadesContext,
queryTableSet, ownerGroup, plan);
queryTableSet, ownerGroup, originalPlan);
if (structInfo != null) {
structInfosBuilder.add(structInfo);
}
@ -197,7 +199,7 @@ public class MaterializedViewUtils {
}
}
// if plan doesn't belong to any group, construct it directly
return ImmutableList.of(StructInfo.of(plan, cascadesContext));
return ImmutableList.of(StructInfo.of(plan, originalPlan, cascadesContext));
}
/**

6
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java
View File

@ -41,7 +41,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
/**
* This record the predicates which can be pulled up or some other type predicates.
@ -70,11 +69,6 @@ public class Predicates {
return new Predicates(mergedPredicates);
}
public Expression composedExpression() {
return ExpressionUtils.and(pulledUpPredicates.stream().map(Expression.class::cast)
.collect(Collectors.toList()));
}
/**
* Split the expression to equal, range and residual predicate.
*/

83
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
View File

@ -106,15 +106,16 @@ public class StructInfo {
// this is for LogicalCompatibilityContext later
private final Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap;
// this recorde the predicates which can pull up, not shuttled
private Predicates predicates;
private final Predicates predicates;
// split predicates is shuttled
private final SplitPredicate splitPredicate;
private final EquivalenceClass equivalenceClass;
private SplitPredicate splitPredicate;
private EquivalenceClass equivalenceClass;
// Key is the expression shuttled and the value is the origin expression
// this is for building LogicalCompatibilityContext later.
private final Map<ExpressionPosition, Map<Expression, Expression>> shuttledExpressionsToExpressionsMap;
// Record the exprId and the corresponding expr map, this is used by expression shuttled
private final Map<ExprId, Expression> namedExprIdAndExprMapping;
private final List<? extends Expression> planOutputShuttledExpressions;
/**
* The construct method for StructInfo
@ -125,30 +126,25 @@ public class StructInfo {
@Nullable Predicates predicates,
Map<ExpressionPosition, Map<Expression, Expression>> shuttledExpressionsToExpressionsMap,
Map<ExprId, Expression> namedExprIdAndExprMapping,
BitSet tableIdSet) {
BitSet tableIdSet,
SplitPredicate splitPredicate,
EquivalenceClass equivalenceClass,
List<? extends Expression> planOutputShuttledExpressions) {
this.originalPlan = originalPlan;
this.originalPlanId = originalPlanId;
this.hyperGraph = hyperGraph;
this.valid = valid
&& hyperGraph.getNodes().stream().allMatch(n -> ((StructInfoNode) n).getExpressions() != null);
this.valid = valid;
this.topPlan = topPlan;
this.bottomPlan = bottomPlan;
this.relations = relations;
this.tableBitSet = tableIdSet;
this.relationIdStructInfoNodeMap = relationIdStructInfoNodeMap;
this.predicates = predicates;
if (predicates == null) {
// collect predicate from top plan which not in hyper graph
Set<Expression> topPlanPredicates = new LinkedHashSet<>();
topPlan.accept(PREDICATE_COLLECTOR, topPlanPredicates);
this.predicates = Predicates.of(topPlanPredicates);
}
Pair<SplitPredicate, EquivalenceClass> derivedPredicates =
predicatesDerive(this.predicates, topPlan, tableBitSet);
this.splitPredicate = derivedPredicates.key();
this.equivalenceClass = derivedPredicates.value();
this.splitPredicate = splitPredicate;
this.equivalenceClass = equivalenceClass;
this.shuttledExpressionsToExpressionsMap = shuttledExpressionsToExpressionsMap;
this.namedExprIdAndExprMapping = namedExprIdAndExprMapping;
this.planOutputShuttledExpressions = planOutputShuttledExpressions;
}
/**
@ -157,7 +153,8 @@ public class StructInfo {
public StructInfo withPredicates(Predicates predicates) {
return new StructInfo(this.originalPlan, this.originalPlanId, this.hyperGraph, this.valid, this.topPlan,
this.bottomPlan, this.relations, this.relationIdStructInfoNodeMap, predicates,
this.shuttledExpressionsToExpressionsMap, this.namedExprIdAndExprMapping, this.tableBitSet);
this.shuttledExpressionsToExpressionsMap, this.namedExprIdAndExprMapping, this.tableBitSet,
null, null, this.planOutputShuttledExpressions);
}
/**
@ -166,7 +163,8 @@ public class StructInfo {
public StructInfo withTableBitSet(BitSet tableBitSet) {
return new StructInfo(this.originalPlan, this.originalPlanId, this.hyperGraph, this.valid, this.topPlan,
this.bottomPlan, this.relations, this.relationIdStructInfoNodeMap, this.predicates,
this.shuttledExpressionsToExpressionsMap, this.namedExprIdAndExprMapping, tableBitSet);
this.shuttledExpressionsToExpressionsMap, this.namedExprIdAndExprMapping, tableBitSet,
this.splitPredicate, this.equivalenceClass, this.planOutputShuttledExpressions);
}
private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph,
@ -252,11 +250,10 @@ public class StructInfo {
}
// derive some useful predicate by predicates
private Pair<SplitPredicate, EquivalenceClass> predicatesDerive(Predicates predicates, Plan originalPlan,
BitSet tableBitSet) {
private static Pair<SplitPredicate, EquivalenceClass> predicatesDerive(Predicates predicates, Plan originalPlan) {
// construct equivalenceClass according to equals predicates
List<Expression> shuttledExpression = ExpressionUtils.shuttleExpressionWithLineage(
new ArrayList<>(predicates.getPulledUpPredicates()), originalPlan, tableBitSet).stream()
new ArrayList<>(predicates.getPulledUpPredicates()), originalPlan, new BitSet()).stream()
.map(Expression.class::cast)
.collect(Collectors.toList());
SplitPredicate splitPredicate = Predicates.splitPredicates(ExpressionUtils.and(shuttledExpression));
@ -328,9 +325,19 @@ public class StructInfo {
relationIdStructInfoNodeMap,
tableBitSet,
cascadesContext);
valid = valid
&& hyperGraph.getNodes().stream().allMatch(n -> ((StructInfoNode) n).getExpressions() != null);
// collect predicate from top plan which not in hyper graph
Set<Expression> topPlanPredicates = new LinkedHashSet<>();
topPlan.accept(PREDICATE_COLLECTOR, topPlanPredicates);
Predicates predicates = Predicates.of(topPlanPredicates);
// this should use the output of originalPlan to make sure the output right order
List<? extends Expression> planOutputShuttledExpressions =
ExpressionUtils.shuttleExpressionWithLineage(originalPlan.getOutput(), originalPlan, new BitSet());
return new StructInfo(originalPlan, originalPlanId, hyperGraph, valid, topPlan, bottomPlan,
relationList, relationIdStructInfoNodeMap, null, shuttledHashConjunctsToConjunctsMap,
namedExprIdAndExprMapping, tableBitSet);
relationList, relationIdStructInfoNodeMap, predicates, shuttledHashConjunctsToConjunctsMap,
namedExprIdAndExprMapping, tableBitSet, null, null,
planOutputShuttledExpressions);
}
/**
@ -350,10 +357,6 @@ public class StructInfo {
return predicates;
}
public EquivalenceClass getEquivalenceClass() {
return equivalenceClass;
}
public Plan getOriginalPlan() {
return originalPlan;
}
@ -362,8 +365,28 @@ public class StructInfo {
return hyperGraph;
}
/**
* lazy init for performance
*/
public SplitPredicate getSplitPredicate() {
return splitPredicate;
if (this.splitPredicate == null && this.predicates != null) {
Pair<SplitPredicate, EquivalenceClass> derivedPredicates = predicatesDerive(this.predicates, topPlan);
this.splitPredicate = derivedPredicates.key();
this.equivalenceClass = derivedPredicates.value();
}
return this.splitPredicate;
}
/**
* lazy init for performance
*/
public EquivalenceClass getEquivalenceClass() {
if (this.equivalenceClass == null && this.predicates != null) {
Pair<SplitPredicate, EquivalenceClass> derivedPredicates = predicatesDerive(this.predicates, topPlan);
this.splitPredicate = derivedPredicates.key();
this.equivalenceClass = derivedPredicates.value();
}
return this.equivalenceClass;
}
public boolean isValid() {
@ -416,6 +439,10 @@ public class StructInfo {
return tableBitSet;
}
public List<? extends Expression> getPlanOutputShuttledExpressions() {
return planOutputShuttledExpressions;
}
/**
* Judge the source graph logical is whether the same as target
* For inner join should judge only the join tables,