database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[feature](mtmv) Support query rewrite by materialized view when query is aggregate and materialized view has no aggregate (#36278) (#37497)

Browse Source 
cherry-pick from master
pr: #36278
commitId: 649f9bc6
This commit is contained in:
seawinde
2024-07-11 10:54:50 +08:00
committed by GitHub
parent e6b8ebc847
commit 1eb04cf538
12 changed files with 890 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
View File

@ -40,6 +40,7 @@ import org.apache.doris.nereids.rules.exploration.join.PushDownProjectThroughInn
import org.apache.doris.nereids.rules.exploration.join.PushDownProjectThroughSemiJoin;
import org.apache.doris.nereids.rules.exploration.join.SemiJoinSemiJoinTranspose;
import org.apache.doris.nereids.rules.exploration.join.SemiJoinSemiJoinTransposeProject;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewAggregateOnNoneAggregateRule;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewAggregateRule;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterAggregateRule;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterJoinRule;
@ -257,6 +258,7 @@ public class RuleSet {
.add(MaterializedViewFilterProjectScanRule.INSTANCE)
.add(MaterializedViewProjectScanRule.INSTANCE)
.add(MaterializedViewProjectFilterScanRule.INSTANCE)
.add(MaterializedViewAggregateOnNoneAggregateRule.INSTANCE)
.build();
public static final List<Rule> DPHYP_REORDER_RULES = ImmutableList.<Rule>builder()

6
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
View File

@ -390,6 +390,12 @@ public enum RuleType {
MATERIALIZED_VIEW_FILTER_PROJECT_AGGREGATE(RuleTypeClass.EXPLORATION),
MATERIALIZED_VIEW_ONLY_AGGREGATE(RuleTypeClass.EXPLORATION),
MATERIALIZED_VIEW_PROJECT_AGGREGATE_ON_NONE_AGGREGATE(RuleTypeClass.EXPLORATION),
MATERIALIZED_VIEW_FILTER_AGGREGATE_ON_NONE_AGGREGATE(RuleTypeClass.EXPLORATION),
MATERIALIZED_VIEW_PROJECT_FILTER_AGGREGATE_ON_NONE_AGGREGATE(RuleTypeClass.EXPLORATION),
MATERIALIZED_VIEW_FILTER_PROJECT_AGGREGATE_ON_NONE_AGGREGATE(RuleTypeClass.EXPLORATION),
MATERIALIZED_VIEW_ONLY_AGGREGATE_ON_NONE_AGGREGATE(RuleTypeClass.EXPLORATION),
MATERIALIZED_VIEW_FILTER_SCAN(RuleTypeClass.EXPLORATION),
MATERIALIZED_VIEW_PROJECT_SCAN(RuleTypeClass.EXPLORATION),
MATERIALIZED_VIEW_FILTER_PROJECT_SCAN(RuleTypeClass.EXPLORATION),

116
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.analysis.NormalizeRepeat;
import org.apache.doris.nereids.rules.exploration.mv.AbstractMaterializedViewAggregateRule.AggregateExpressionRewriteContext.ExpressionRewriteMode;
import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext;
import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanSplitContext;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
@ -136,6 +137,27 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
materializationContext.getShuttledExprToScanExprMapping(),
viewToQuerySlotMapping));
}
return doRewriteQueryByView(queryStructInfo,
viewToQuerySlotMapping,
queryTopPlanAndAggPair,
tempRewritedPlan,
materializationContext,
ExpressionRewriteMode.EXPRESSION_DIRECT,
ExpressionRewriteMode.EXPRESSION_ROLL_UP);
}
/**
* Aggregate function and group by expression rewrite impl
*/
protected LogicalAggregate<Plan> doRewriteQueryByView(
StructInfo queryStructInfo,
SlotMapping viewToQuerySlotMapping,
Pair<Plan, LogicalAggregate<Plan>> queryTopPlanAndAggPair,
Plan tempRewritedPlan,
MaterializationContext materializationContext,
ExpressionRewriteMode groupByMode,
ExpressionRewriteMode aggregateFunctionMode) {
// try to roll up.
// split the query top plan expressions to group expressions and functions, if can not, bail out.
Pair<Set<? extends Expression>, Set<? extends Expression>> queryGroupAndFunctionPair
@ -149,11 +171,12 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
Map<Expression, Expression> mvExprToMvScanExprQueryBased =
materializationContext.getShuttledExprToScanExprMapping().keyPermute(viewToQuerySlotMapping)
.flattenMap().get(0);
Plan queryTopPlan = queryStructInfo.getTopPlan();
for (Expression topExpression : queryTopPlan.getOutput()) {
if (queryTopPlanFunctionSet.contains(topExpression)) {
// if agg function, try to roll up and rewrite
Expression rollupedExpression = tryRewriteExpression(queryStructInfo, topExpression,
mvExprToMvScanExprQueryBased, false, materializationContext,
mvExprToMvScanExprQueryBased, aggregateFunctionMode, materializationContext,
"Query function roll up fail",
() -> String.format("queryExpression = %s,\n mvExprToMvScanExprQueryBased = %s",
topExpression, mvExprToMvScanExprQueryBased));
@ -164,7 +187,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
} else {
// if group by dimension, try to rewrite
Expression rewrittenGroupByExpression = tryRewriteExpression(queryStructInfo, topExpression,
mvExprToMvScanExprQueryBased, true, materializationContext,
mvExprToMvScanExprQueryBased, groupByMode, materializationContext,
"View dimensions doesn't not cover the query dimensions",
() -> String.format("mvExprToMvScanExprQueryBased is %s,\n queryExpression is %s",
mvExprToMvScanExprQueryBased, topExpression));
@ -178,6 +201,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
finalGroupExpressions.add(groupByExpression);
}
}
LogicalAggregate<Plan> queryAggregate = queryTopPlanAndAggPair.value();
List<Expression> queryGroupByExpressions = queryAggregate.getGroupByExpressions();
// handle the scene that query top plan not use the group by in query bottom aggregate
if (queryGroupByExpressions.size() != queryTopPlanGroupBySet.size()) {
@ -186,7 +210,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
continue;
}
Expression rewrittenGroupByExpression = tryRewriteExpression(queryStructInfo, expression,
mvExprToMvScanExprQueryBased, true, materializationContext,
mvExprToMvScanExprQueryBased, groupByMode, materializationContext,
"View dimensions doesn't not cover the query dimensions in bottom agg ",
() -> String.format("mvExprToMvScanExprQueryBased is %s,\n expression is %s",
mvExprToMvScanExprQueryBased, expression));
@ -198,7 +222,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
finalGroupExpressions.add(groupByExpression);
}
}
if (queryContainsGroupSets) {
if (queryAggregate.getSourceRepeat().isPresent()) {
// construct group sets for repeat
List<List<Expression>> rewrittenGroupSetsExpressions = new ArrayList<>();
List<List<Expression>> groupingSets = queryAggregate.getSourceRepeat().get().getGroupingSets();
@ -209,7 +233,8 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
List<Expression> rewrittenGroupSetExpressions = new ArrayList<>();
for (Expression expression : groupingSet) {
Expression rewrittenGroupByExpression = tryRewriteExpression(queryStructInfo, expression,
mvExprToMvScanExprQueryBased, true, materializationContext,
mvExprToMvScanExprQueryBased, ExpressionRewriteMode.EXPRESSION_DIRECT,
materializationContext,
"View dimensions doesn't not cover the query group set dimensions",
() -> String.format("mvExprToMvScanExprQueryBased is %s,\n queryExpression is %s",
mvExprToMvScanExprQueryBased, expression));
@ -232,14 +257,14 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
* Try to rewrite query expression by view, contains both group by dimension and aggregate function
*/
protected Expression tryRewriteExpression(StructInfo queryStructInfo, Expression queryExpression,
Map<Expression, Expression> mvShuttledExprToMvScanExprQueryBased, boolean isGroupBy,
Map<Expression, Expression> mvShuttledExprToMvScanExprQueryBased, ExpressionRewriteMode rewriteMode,
MaterializationContext materializationContext, String summaryIfFail, Supplier<String> detailIfFail) {
Expression queryFunctionShuttled = ExpressionUtils.shuttleExpressionWithLineage(
queryExpression,
queryStructInfo.getTopPlan(),
queryStructInfo.getTableBitSet());
AggregateExpressionRewriteContext expressionRewriteContext = new AggregateExpressionRewriteContext(
isGroupBy, mvShuttledExprToMvScanExprQueryBased, queryStructInfo.getTopPlan(),
rewriteMode, mvShuttledExprToMvScanExprQueryBased, queryStructInfo.getTopPlan(),
queryStructInfo.getTableBitSet());
Expression rewrittenExpression = queryFunctionShuttled.accept(AGGREGATE_EXPRESSION_REWRITER,
expressionRewriteContext);
@ -344,7 +369,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
return null;
}
private Pair<Set<? extends Expression>, Set<? extends Expression>> topPlanSplitToGroupAndFunction(
protected Pair<Set<? extends Expression>, Set<? extends Expression>> topPlanSplitToGroupAndFunction(
Pair<Plan, LogicalAggregate<Plan>> topPlanAndAggPair, StructInfo queryStructInfo) {
LogicalAggregate<Plan> bottomQueryAggregate = topPlanAndAggPair.value();
Set<Expression> groupByExpressionSet = new HashSet<>(bottomQueryAggregate.getGroupByExpressions());
@ -377,7 +402,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
return Pair.of(topGroupByExpressions, topFunctionExpressions);
}
private Pair<Plan, LogicalAggregate<Plan>> splitToTopPlanAndAggregate(StructInfo structInfo) {
protected Pair<Plan, LogicalAggregate<Plan>> splitToTopPlanAndAggregate(StructInfo structInfo) {
Plan topPlan = structInfo.getTopPlan();
PlanSplitContext splitContext = new PlanSplitContext(Sets.newHashSet(LogicalAggregate.class));
topPlan.accept(StructInfo.PLAN_SPLITTER, splitContext);
@ -394,7 +419,7 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
* slot reference equals currently.
*/
@Override
protected boolean checkPattern(StructInfo structInfo, CascadesContext cascadesContext) {
protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext cascadesContext) {
PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET);
// if query or mv contains more then one top aggregate, should fail
return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext)
@ -414,17 +439,38 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
if (!rewriteContext.isValid()) {
return aggregateFunction;
}
Expression queryFunctionShuttled = ExpressionUtils.shuttleExpressionWithLineage(
aggregateFunction,
rewriteContext.getQueryTopPlan(),
rewriteContext.getQueryTableBitSet());
Function rollupAggregateFunction = rollup(aggregateFunction, queryFunctionShuttled,
rewriteContext.getMvExprToMvScanExprQueryBasedMapping());
if (rollupAggregateFunction == null) {
if (ExpressionRewriteMode.EXPRESSION_DIRECT.equals(rewriteContext.getExpressionRewriteMode())) {
rewriteContext.setValid(false);
return aggregateFunction;
}
return rollupAggregateFunction;
Function rewrittenFunction;
if (ExpressionRewriteMode.EXPRESSION_ROLL_UP.equals(rewriteContext.getExpressionRewriteMode())) {
Expression queryFunctionShuttled = ExpressionUtils.shuttleExpressionWithLineage(
aggregateFunction,
rewriteContext.getQueryTopPlan(),
rewriteContext.getQueryTableBitSet());
rewrittenFunction = rollup(aggregateFunction, queryFunctionShuttled,
rewriteContext.getMvExprToMvScanExprQueryBasedMapping());
if (rewrittenFunction == null) {
rewriteContext.setValid(false);
return aggregateFunction;
}
return rewrittenFunction;
}
if (ExpressionRewriteMode.EXPRESSION_DIRECT_ALL.equals(rewriteContext.getExpressionRewriteMode())) {
List<Expression> children = aggregateFunction.children();
List<Expression> rewrittenChildren = new ArrayList<>();
for (Expression child : children) {
Expression rewrittenExpression = child.accept(this, rewriteContext);
if (!rewriteContext.isValid()) {
return aggregateFunction;
}
rewrittenChildren.add(rewrittenExpression);
}
return aggregateFunction.withChildren(rewrittenChildren);
}
rewriteContext.setValid(false);
return aggregateFunction;
}
@Override
@ -474,7 +520,8 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
return expr;
}
// for group by expression try to get corresponding expression directly
if (rewriteContext.isOnlyContainGroupByExpression()
if ((ExpressionRewriteMode.EXPRESSION_DIRECT.equals(rewriteContext.getExpressionRewriteMode())
|| ExpressionRewriteMode.EXPRESSION_DIRECT_ALL.equals(rewriteContext.getExpressionRewriteMode()))
&& rewriteContext.getMvExprToMvScanExprQueryBasedMapping().containsKey(expr)) {
return rewriteContext.getMvExprToMvScanExprQueryBasedMapping().get(expr);
}
@ -499,15 +546,15 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
*/
protected static class AggregateExpressionRewriteContext {
private boolean valid = true;
private final boolean onlyContainGroupByExpression;
private final ExpressionRewriteMode expressionRewriteMode;
private final Map<Expression, Expression> mvExprToMvScanExprQueryBasedMapping;
private final Plan queryTopPlan;
private final BitSet queryTableBitSet;
public AggregateExpressionRewriteContext(boolean onlyContainGroupByExpression,
public AggregateExpressionRewriteContext(ExpressionRewriteMode expressionRewriteMode,
Map<Expression, Expression> mvExprToMvScanExprQueryBasedMapping, Plan queryTopPlan,
BitSet queryTableBitSet) {
this.onlyContainGroupByExpression = onlyContainGroupByExpression;
this.expressionRewriteMode = expressionRewriteMode;
this.mvExprToMvScanExprQueryBasedMapping = mvExprToMvScanExprQueryBasedMapping;
this.queryTopPlan = queryTopPlan;
this.queryTableBitSet = queryTableBitSet;
@ -521,8 +568,8 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
this.valid = valid;
}
public boolean isOnlyContainGroupByExpression() {
return onlyContainGroupByExpression;
public ExpressionRewriteMode getExpressionRewriteMode() {
return expressionRewriteMode;
}
public Map<Expression, Expression> getMvExprToMvScanExprQueryBasedMapping() {
@ -536,5 +583,26 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
public BitSet getQueryTableBitSet() {
return queryTableBitSet;
}
/**
* The expression rewrite mode, which decide how the expression in query is rewritten by mv
*/
protected enum ExpressionRewriteMode {
/**
* Try to use the expression in mv directly, and doesn't handle aggregate function
*/
EXPRESSION_DIRECT,
/**
* Try to use the expression in mv directly, and try to rewrite the arguments in aggregate function except
* the aggregate function
*/
EXPRESSION_DIRECT_ALL,
/**
* Try to roll up aggregate function
*/
EXPRESSION_ROLL_UP
}
}
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java
View File

@ -75,7 +75,7 @@ public abstract class AbstractMaterializedViewJoinRule extends AbstractMateriali
* Join condition should be slot reference equals currently.
*/
@Override
protected boolean checkPattern(StructInfo structInfo, CascadesContext cascadesContext) {
protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext cascadesContext) {
PlanCheckContext checkContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET);
return structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, checkContext)
&& !checkContext.isContainsTopAggregate();

18
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View File

@ -43,7 +43,6 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.VirtualSlotReference;
import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable;
@ -149,7 +148,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
List<StructInfo> uncheckedStructInfos = MaterializedViewUtils.extractStructInfo(queryPlan, cascadesContext,
materializedViewTableSet);
uncheckedStructInfos.forEach(queryStructInfo -> {
boolean valid = checkPattern(queryStructInfo, cascadesContext) && queryStructInfo.isValid();
boolean valid = checkQueryPattern(queryStructInfo, cascadesContext) && queryStructInfo.isValid();
if (!valid) {
cascadesContext.getMaterializationContexts().forEach(ctx ->
ctx.recordFailReason(queryStructInfo, "Query struct info is invalid",
@ -346,7 +345,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
}
}
private boolean needUnionRewrite(
protected boolean needUnionRewrite(
Pair<Map<BaseTableInfo, Set<String>>, Map<BaseTableInfo, Set<String>>> invalidPartitions,
CascadesContext cascadesContext) {
return invalidPartitions != null
@ -689,13 +688,17 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
/**
* Check the pattern of query or materializedView is supported or not.
*/
protected boolean checkPattern(StructInfo structInfo, CascadesContext cascadesContext) {
protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext cascadesContext) {
if (structInfo.getRelations().isEmpty()) {
return false;
}
return true;
}
protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesContext cascadesContext) {
return checkQueryPattern(structInfo, cascadesContext);
}
protected void recordIfRewritten(Plan plan, MaterializationContext context) {
context.setSuccess(true);
if (plan.getGroupExpression().isPresent()) {
@ -708,11 +711,6 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
&& context.alreadyRewrite(plan.getGroupExpression().get().getOwnerGroup().getGroupId());
}
protected boolean isEmptyVirtualSlot(Expression expression) {
return expression instanceof VirtualSlotReference
&& ((VirtualSlotReference) expression).getRealExpressions().isEmpty();
}
// check mv plan is valid or not, this can use cache for performance
private boolean isMaterializationValid(CascadesContext cascadesContext, MaterializationContext context) {
long materializationId = context.getMaterializationQualifier().hashCode();
@ -720,7 +718,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
materializationId);
if (cachedCheckResult == null) {
// need check in real time
boolean checkResult = checkPattern(context.getStructInfo(), cascadesContext);
boolean checkResult = checkMaterializationPattern(context.getStructInfo(), cascadesContext);
if (!checkResult) {
context.recordFailReason(context.getStructInfo(),
"View struct info is invalid", () -> String.format("view plan is %s",

134
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java Normal file
View File

@ -0,0 +1,134 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Pair;
import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.exploration.mv.AbstractMaterializedViewAggregateRule.AggregateExpressionRewriteContext.ExpressionRewriteMode;
import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* MaterializedViewAggregateOnNoAggregateRule
*/
public class MaterializedViewAggregateOnNoneAggregateRule extends AbstractMaterializedViewAggregateRule {
public static final MaterializedViewAggregateOnNoneAggregateRule INSTANCE =
new MaterializedViewAggregateOnNoneAggregateRule();
@Override
public List<Rule> buildRules() {
return ImmutableList.of(
logicalFilter(logicalProject(logicalAggregate(any().when(LogicalPlan.class::isInstance))))
.thenApplyMultiNoThrow(ctx -> {
LogicalFilter<LogicalProject<LogicalAggregate<Plan>>> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_FILTER_PROJECT_AGGREGATE_ON_NONE_AGGREGATE),
logicalAggregate(any().when(LogicalPlan.class::isInstance)).thenApplyMultiNoThrow(ctx -> {
LogicalAggregate<Plan> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_ONLY_AGGREGATE_ON_NONE_AGGREGATE),
logicalProject(logicalFilter(logicalAggregate(any().when(LogicalPlan.class::isInstance))))
.thenApplyMultiNoThrow(ctx -> {
LogicalProject<LogicalFilter<LogicalAggregate<Plan>>> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_FILTER_AGGREGATE_ON_NONE_AGGREGATE),
logicalProject(logicalAggregate(any().when(LogicalPlan.class::isInstance))).thenApplyMultiNoThrow(
ctx -> {
LogicalProject<LogicalAggregate<Plan>> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_AGGREGATE_ON_NONE_AGGREGATE),
logicalFilter(logicalAggregate(any().when(LogicalPlan.class::isInstance))).thenApplyMultiNoThrow(
ctx -> {
LogicalFilter<LogicalAggregate<Plan>> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_FILTER_AGGREGATE_ON_NONE_AGGREGATE));
}
@Override
protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesContext cascadesContext) {
// any check result of join or scan is true, then return true
PlanCheckContext joinCheckContext = PlanCheckContext.of(SUPPORTED_JOIN_TYPE_SET);
boolean joinCheckResult = structInfo.getTopPlan().accept(StructInfo.PLAN_PATTERN_CHECKER, joinCheckContext)
&& !joinCheckContext.isContainsTopAggregate();
if (joinCheckResult) {
return true;
}
PlanCheckContext scanCheckContext = PlanCheckContext.of(ImmutableSet.of());
return structInfo.getTopPlan().accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, scanCheckContext)
&& !scanCheckContext.isContainsTopAggregate();
}
@Override
protected Pair<Map<BaseTableInfo, Set<String>>, Map<BaseTableInfo, Set<String>>> calcInvalidPartitions(
Plan queryPlan, Plan rewrittenPlan, AsyncMaterializationContext materializationContext,
CascadesContext cascadesContext) throws AnalysisException {
Pair<Map<BaseTableInfo, Set<String>>, Map<BaseTableInfo, Set<String>>> invalidPartitions
= super.calcInvalidPartitions(queryPlan, rewrittenPlan, materializationContext, cascadesContext);
if (needUnionRewrite(invalidPartitions, cascadesContext)) {
// if query use some invalid partition in mv, bail out
return null;
}
return invalidPartitions;
}
@Override
protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInfo, StructInfo viewStructInfo,
SlotMapping viewToQuerySlotMapping, Plan tempRewritedPlan, MaterializationContext materializationContext) {
// check the expression used in group by and group out expression in query
Pair<Plan, LogicalAggregate<Plan>> queryTopPlanAndAggPair = splitToTopPlanAndAggregate(queryStructInfo);
if (queryTopPlanAndAggPair == null) {
materializationContext.recordFailReason(queryStructInfo,
"Split query to top plan and agg fail",
() -> String.format("query plan = %s\n", queryStructInfo.getOriginalPlan().treeString()));
return null;
}
LogicalAggregate<Plan> queryAggregate = queryTopPlanAndAggPair.value();
boolean queryContainsGroupSets = queryAggregate.getSourceRepeat().isPresent();
if (queryContainsGroupSets) {
// doesn't support group sets momentarily
materializationContext.recordFailReason(queryStructInfo,
"Query function roll up fail",
() -> String.format("query aggregate = %s", queryAggregate.treeString()));
return null;
}
return doRewriteQueryByView(queryStructInfo,
viewToQuerySlotMapping,
queryTopPlanAndAggPair,
tempRewritedPlan,
materializationContext,
ExpressionRewriteMode.EXPRESSION_DIRECT_ALL,
ExpressionRewriteMode.EXPRESSION_DIRECT_ALL);
}
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectAggregateRule.java
View File

@ -44,6 +44,6 @@ public class MaterializedViewFilterProjectAggregateRule extends AbstractMaterial
.thenApplyMultiNoThrow(ctx -> {
LogicalFilter<LogicalProject<LogicalAggregate<Plan>>> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_FILTER_AGGREGATE));
}).toRule(RuleType.MATERIALIZED_VIEW_FILTER_PROJECT_AGGREGATE));
}
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterAggregateRule.java
View File

@ -43,6 +43,6 @@ public class MaterializedViewProjectFilterAggregateRule extends AbstractMaterial
any().when(LogicalPlan.class::isInstance)))).thenApplyMultiNoThrow(ctx -> {
LogicalProject<LogicalFilter<LogicalAggregate<Plan>>> root = ctx.root;
return rewrite(root, ctx.cascadesContext);
}).toRule(RuleType.MATERIALIZED_VIEW_FILTER_AGGREGATE));
}).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_FILTER_AGGREGATE));
}
}

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java
View File

@ -76,7 +76,7 @@ public abstract class MaterializedViewScanRule extends AbstractMaterializedViewR
* Join condition should be slot reference equals currently.
*/
@Override
protected boolean checkPattern(StructInfo structInfo, CascadesContext cascadesContext) {
protected boolean checkQueryPattern(StructInfo structInfo, CascadesContext cascadesContext) {
PlanCheckContext checkContext = PlanCheckContext.of(ImmutableSet.of());
return structInfo.getTopPlan().accept(StructInfo.SCAN_PLAN_PATTERN_CHECKER, checkContext)
&& !checkContext.isContainsTopAggregate();