database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Test](Nereids) Add SSB dataset to test materialized view rewrite (#30528)

Browse Source 
* [Test](Nereids) Add SSB dataset to test materialized view rewrite

* rollback irrelevant code

* fix sort slot 0
This commit is contained in:
seawinde
2024-02-01 13:39:29 +08:00
committed by yiguolei
parent 2a81ab70e4
commit 1ab37737ae
20 changed files with 4667 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java
View File

@ -72,7 +72,7 @@ public class MTMVCache {
? (Plan) ((LogicalResultSink) mvRewrittenPlan).child() : mvRewrittenPlan;
// use rewritten plan output expression currently, if expression rewrite fail,
// consider to use the analyzed plan for output expressions only
List<NamedExpression> mvOutputExpressions = mvPlan.getExpressions().stream()
List<NamedExpression> mvOutputExpressions = mvPlan.getOutput().stream()
.map(NamedExpression.class::cast)
.collect(Collectors.toList());
return new MTMVCache(mvPlan, mvOutputExpressions);

48
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
View File

@ -42,10 +42,13 @@ import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewri
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer;
import org.apache.doris.nereids.types.BigIntType;
import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
@ -118,7 +121,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
// Firstly,if group by expression between query and view is equals, try to rewrite expression directly
Plan queryTopPlan = queryTopPlanAndAggPair.key();
if (isGroupByEquals(queryTopPlanAndAggPair, viewTopPlanAndAggPair, viewToQuerySlotMapping)) {
List<Expression> rewrittenQueryExpressions = rewriteExpression(queryTopPlan.getExpressions(),
List<Expression> rewrittenQueryExpressions = rewriteExpression(queryTopPlan.getOutput(),
queryTopPlan,
materializationContext.getMvExprToMvScanExprMapping(),
viewToQuerySlotMapping,
@ -135,7 +138,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
Pair.of("Can not rewrite expression when no roll up",
String.format("expressionToWrite = %s,\n mvExprToMvScanExprMapping = %s,\n"
+ "viewToQuerySlotMapping = %s",
queryTopPlan.getExpressions(),
queryTopPlan.getOutput(),
materializationContext.getMvExprToMvScanExprMapping(),
viewToQuerySlotMapping)));
}
@ -161,7 +164,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
// try to rewrite, contains both roll up aggregate functions and aggregate group expression
List<NamedExpression> finalOutputExpressions = new ArrayList<>();
List<Expression> finalGroupExpressions = new ArrayList<>();
List<? extends Expression> queryExpressions = queryTopPlan.getExpressions();
List<? extends Expression> queryExpressions = queryTopPlan.getOutput();
// permute the mv expr mapping to query based
Map<Expression, Expression> mvExprToMvScanExprQueryBased =
materializationContext.getMvExprToMvScanExprMapping().keyPermute(viewToQuerySlotMapping)
@ -231,13 +234,8 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
})
.collect(Collectors.toList());
finalOutputExpressions = finalOutputExpressions.stream()
.map(expr -> {
ExprId exprId = expr.getExprId();
if (projectOutPutExprIdMap.containsKey(exprId)) {
return projectOutPutExprIdMap.get(exprId);
}
return expr;
})
.map(expr -> projectOutPutExprIdMap.containsKey(expr.getExprId())
? projectOutPutExprIdMap.get(expr.getExprId()) : expr)
.collect(Collectors.toList());
return new LogicalAggregate(finalGroupExpressions, finalOutputExpressions, mvProject);
}
@ -327,24 +325,29 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
private Pair<Set<? extends Expression>, Set<? extends Expression>> topPlanSplitToGroupAndFunction(
Pair<Plan, LogicalAggregate<Plan>> topPlanAndAggPair) {
LogicalAggregate<Plan> queryAggregate = topPlanAndAggPair.value();
Set<Expression> queryAggGroupSet = new HashSet<>(queryAggregate.getGroupByExpressions());
LogicalAggregate<Plan> bottomQueryAggregate = topPlanAndAggPair.value();
Set<Expression> groupByExpressionSet = new HashSet<>(bottomQueryAggregate.getGroupByExpressions());
// when query is bitmap_count(bitmap_union), the plan is as following:
// project(bitmap_count()#1)
// aggregate(bitmap_union()#2)
// we should use exprId which query top plan used to decide the query top plan is use the
// bottom agg function or not
Set<ExprId> queryAggFunctionSet = queryAggregate.getOutputExpressions().stream()
.filter(expr -> !queryAggGroupSet.contains(expr))
Set<ExprId> bottomAggregateFunctionExprIdSet = bottomQueryAggregate.getOutput().stream()
.filter(expr -> !groupByExpressionSet.contains(expr))
.map(NamedExpression::getExprId)
.collect(Collectors.toSet());
Plan queryTopPlan = topPlanAndAggPair.key();
Set<Expression> topGroupByExpressions = new HashSet<>();
Set<Expression> topFunctionExpressions = new HashSet<>();
queryTopPlan.getExpressions().forEach(expression -> {
if (expression.anyMatch(expr -> expr instanceof NamedExpression
&& queryAggFunctionSet.contains(((NamedExpression) expr).getExprId()))) {
queryTopPlan.getOutput().forEach(expression -> {
ExpressionLineageReplacer.ExpressionReplaceContext replaceContext =
new ExpressionLineageReplacer.ExpressionReplaceContext(ImmutableList.of(expression),
ImmutableSet.of(), ImmutableSet.of());
queryTopPlan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext);
if (!Sets.intersection(bottomAggregateFunctionExprIdSet,
replaceContext.getExprIdExpressionMap().keySet()).isEmpty()) {
// if query top plan expression use any aggregate function, then consider it is aggregate function
topFunctionExpressions.add(expression);
} else {
topGroupByExpressions.add(expression);
@ -378,14 +381,13 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
HyperGraph hyperGraph = structInfo.getHyperGraph();
for (AbstractNode node : hyperGraph.getNodes()) {
StructInfoNode structInfoNode = (StructInfoNode) node;
if (!structInfoNode.getPlan().accept(StructInfo.JOIN_PATTERN_CHECKER,
SUPPORTED_JOIN_TYPE_SET)) {
if (!structInfoNode.getPlan().accept(StructInfo.JOIN_PATTERN_CHECKER, SUPPORTED_JOIN_TYPE_SET)) {
return false;
}
for (JoinEdge edge : hyperGraph.getJoinEdges()) {
if (!edge.getJoin().accept(StructInfo.JOIN_PATTERN_CHECKER, SUPPORTED_JOIN_TYPE_SET)) {
return false;
}
}
for (JoinEdge edge : hyperGraph.getJoinEdges()) {
if (!edge.getJoin().accept(StructInfo.JOIN_PATTERN_CHECKER, SUPPORTED_JOIN_TYPE_SET)) {
return false;
}
}
return true;

6
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java
View File

@ -29,9 +29,6 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.List;
import java.util.stream.Collectors;
@ -41,9 +38,6 @@ import java.util.stream.Collectors;
*/
public abstract class AbstractMaterializedViewJoinRule extends AbstractMaterializedViewRule {
protected final String currentClassName = this.getClass().getSimpleName();
private final Logger logger = LogManager.getLogger(this.getClass());
@Override
protected Plan rewriteQueryByView(MatchMode matchMode,
StructInfo queryStructInfo,

4
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
View File

@ -195,6 +195,10 @@ public class MaterializedViewUtils {
@Override
public Void visitLogicalJoin(LogicalJoin<? extends Plan, ? extends Plan> join,
IncrementCheckerContext context) {
if (join.isMarkJoin()) {
context.setPctPossible(false);
return null;
}
Plan left = join.child(0);
Set<Column> leftColumnSet = left.getOutputSet().stream()
.filter(slot -> slot instanceof SlotReference

4
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
View File

@ -37,6 +37,7 @@ import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
import org.apache.doris.nereids.trees.plans.algebra.Filter;
import org.apache.doris.nereids.trees.plans.algebra.Join;
import org.apache.doris.nereids.trees.plans.algebra.Project;
import org.apache.doris.nereids.trees.plans.algebra.Sort;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
@ -457,6 +458,7 @@ public class StructInfo {
&& !(plan instanceof Project)
&& !(plan instanceof CatalogRelation)
&& !(plan instanceof Join)
&& !(plan instanceof Sort)
&& !(plan instanceof LogicalAggregate && !((LogicalAggregate) plan).getSourceRepeat()
.isPresent())) {
return false;
@ -489,7 +491,7 @@ public class StructInfo {
super.visit(aggregate, context);
return true;
}
if (plan instanceof LogicalProject || plan instanceof LogicalFilter) {
if (plan instanceof Project || plan instanceof Filter || plan instanceof Sort) {
super.visit(plan, context);
return true;
}