database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Bug](materialized-view) fix wrong result when salias name same with base slot on mv (#33198)

Browse Source 
fix wrong result when salias name same with base slot on mv
This commit is contained in:
Pxl
2024-04-07 10:53:29 +08:00
committed by yiguolei
parent 0e99926b28
commit 2092a862fc
5 changed files with 136 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java
View File

@ -460,9 +460,6 @@ public abstract class AbstractSelectMaterializedIndexRule {
for (Slot mvSlot : mvPlan.getOutputByIndex(mvPlan.getSelectedIndexId())) {
boolean isPushed = false;
for (Slot baseSlot : mvPlan.getOutput()) {
if (org.apache.doris.analysis.CreateMaterializedViewStmt.isMVColumn(mvSlot.getName())) {
continue;
}
if (baseSlot.toSql().equalsIgnoreCase(
org.apache.doris.analysis.CreateMaterializedViewStmt.mvColumnBreaker(
normalizeName(mvSlot.getName())))) {
@ -472,11 +469,6 @@ public abstract class AbstractSelectMaterializedIndexRule {
}
}
if (!isPushed) {
if (org.apache.doris.analysis.CreateMaterializedViewStmt.isMVColumn(mvSlot.getName())) {
mvNameToMvSlot.put(normalizeName(
org.apache.doris.analysis.CreateMaterializedViewStmt.mvColumnBreaker(mvSlot.getName())),
mvSlot);
}
mvNameToMvSlot.put(normalizeName(mvSlot.getName()), mvSlot);
}
}

118
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java
View File

@ -355,18 +355,16 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
return new LogicalProject<>(
generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(
return new LogicalProject<>(generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext).replace(new LogicalAggregate<>(
agg.getGroupByExpressions(),
replaceAggOutput(
agg, Optional.empty(), Optional.empty(), result.exprRewriteMap),
agg.isNormalized(),
agg.getSourceRepeat(),
agg, Optional.empty(), Optional.empty(), result.exprRewriteMap),
agg.isNormalized(), agg.getSourceRepeat(),
repeat.withAggOutputAndChild(
generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), mvPlan)
), mvPlan));
replaceRepeatOutput(repeat, result.exprRewriteMap.projectExprMap),
mvPlan)),
mvPlan));
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_SCAN),
// filter could push down scan.
@ -400,21 +398,18 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.map(e -> result.exprRewriteMap.replaceAgg(e)).collect(Collectors.toSet()),
filter.getConjuncts());
return new LogicalProject<>(
generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(
return new LogicalProject<>(generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext).replace(new LogicalAggregate<>(
agg.getGroupByExpressions(),
replaceAggOutput(agg, Optional.empty(), Optional.empty(),
result.exprRewriteMap),
agg.isNormalized(),
agg.getSourceRepeat(),
agg.isNormalized(), agg.getSourceRepeat(),
// Not that no need to replace slots in the filter,
// because the slots to replace
// are value columns, which shouldn't appear in filters.
repeat.withAggOutputAndChild(
generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()),
filter.withChildren(mvPlan))
replaceRepeatOutput(repeat, result.exprRewriteMap.projectExprMap),
filter.withChildren(mvPlan))
), mvPlan));
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_FILTER_SCAN),
@ -443,21 +438,17 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
List<NamedExpression> newProjectList = replaceProjectList(project,
result.exprRewriteMap.projectExprMap);
LogicalProject<LogicalOlapScan> newProject = new LogicalProject<>(
generateNewOutputsWithMvOutputs(mvPlan, newProjectList),
mvPlan);
return new LogicalProject<>(
generateProjectsAlias(agg.getOutputs(), slotContext),
generateNewOutputsWithMvOutputs(mvPlan, newProjectList), mvPlan);
return new LogicalProject<>(generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(
agg.getGroupByExpressions(),
replaceAggOutput(agg, Optional.of(project), Optional.of(newProject),
result.exprRewriteMap),
agg.isNormalized(),
agg.getSourceRepeat(),
repeat.withAggOutputAndChild(
generateNewOutputsWithMvOutputs(
mvPlan, repeat.getOutputs()), newProject)
), mvPlan));
new LogicalAggregate<>(agg.getGroupByExpressions(),
replaceAggOutput(agg, Optional.of(project), Optional.of(newProject),
result.exprRewriteMap),
agg.isNormalized(), agg.getSourceRepeat(),
repeat.withAggOutputAndChild(replaceRepeatOutput(repeat,
result.exprRewriteMap.projectExprMap), newProject)),
mvPlan));
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_PROJECT_SCAN),
// filter could push down and project.
@ -499,19 +490,15 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
generateNewOutputsWithMvOutputs(mvPlan, newProjectList),
filter.withChildren(mvPlan));
return new LogicalProject<>(
generateProjectsAlias(agg.getOutputs(), slotContext),
return new LogicalProject<>(generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(
agg.getGroupByExpressions(),
replaceAggOutput(agg, Optional.of(project), Optional.of(newProject),
result.exprRewriteMap),
agg.isNormalized(),
agg.getSourceRepeat(),
repeat.withAggOutputAndChild(
generateNewOutputsWithMvOutputs(
mvPlan, repeat.getOutputs()), newProject)
), mvPlan));
new LogicalAggregate<>(agg.getGroupByExpressions(),
replaceAggOutput(agg, Optional.of(project), Optional.of(newProject),
result.exprRewriteMap),
agg.isNormalized(), agg.getSourceRepeat(),
repeat.withAggOutputAndChild(replaceRepeatOutput(repeat,
result.exprRewriteMap.projectExprMap), newProject)),
mvPlan));
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_PROJECT_FILTER_SCAN),
// filter can't push down
@ -551,19 +538,15 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
generateNewOutputsWithMvOutputs(mvPlan, newProjectList),
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId));
return new LogicalProject<>(
generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(
agg.getGroupByExpressions(),
replaceAggOutput(agg, Optional.of(project), Optional.of(newProject),
result.exprRewriteMap),
agg.isNormalized(),
agg.getSourceRepeat(),
return new LogicalProject<>(generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext).replace(new LogicalAggregate<>(
agg.getGroupByExpressions(), replaceAggOutput(agg, Optional.of(project),
Optional.of(newProject), result.exprRewriteMap),
agg.isNormalized(), agg.getSourceRepeat(),
repeat.withAggOutputAndChild(
generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()),
filter.withChildren(newProject))
), mvPlan));
replaceRepeatOutput(repeat, result.exprRewriteMap.projectExprMap),
filter.withChildren(newProject))),
mvPlan));
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_FILTER_PROJECT_SCAN)
);
}
@ -614,22 +597,22 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.stream()
.collect(Collectors.groupingBy(index -> index.getId() == table.getBaseIndexId()));
Set<MaterializedIndex> candidatesWithoutRewriting = indexesGroupByIsBaseOrNot
.getOrDefault(false, ImmutableList.of()).stream()
.filter(index -> preAggEnabledByHint(scan)
|| checkPreAggStatus(scan, index.getId(), predicates, aggregateFunctions, groupingExprs).isOn())
.collect(Collectors.toSet());
// try to rewrite bitmap, hll by materialized index columns.
List<AggRewriteResult> candidatesWithRewriting = indexesGroupByIsBaseOrNot
Set<AggRewriteResult> candidatesWithRewriting = indexesGroupByIsBaseOrNot
.getOrDefault(false, ImmutableList.of()).stream()
.filter(index -> !candidatesWithoutRewriting.contains(index))
.map(index -> rewriteAgg(index, scan, nonVirtualRequiredScanOutput, predicates, aggregateFunctions,
groupingExprs))
.filter(aggRewriteResult -> checkPreAggStatus(scan, aggRewriteResult.index.getId(), predicates,
// check pre-agg status of aggregate function that couldn't rewrite.
aggFuncsDiff(aggregateFunctions, aggRewriteResult), groupingExprs).isOn())
.collect(Collectors.toList());
.collect(Collectors.toSet());
Set<MaterializedIndex> candidatesWithoutRewriting = indexesGroupByIsBaseOrNot
.getOrDefault(false, ImmutableList.of()).stream()
.filter(index -> !candidatesWithRewriting.contains(index))
.filter(index -> preAggEnabledByHint(scan)
|| checkPreAggStatus(scan, index.getId(), predicates, aggregateFunctions, groupingExprs).isOn())
.collect(Collectors.toSet());
List<MaterializedIndex> haveAllRequiredColumns = Streams.concat(
candidatesWithoutRewriting.stream()
@ -1595,6 +1578,13 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
.collect(Collectors.toList());
}
private List<NamedExpression> replaceRepeatOutput(LogicalRepeat<? extends Plan> repeat,
Map<Expression, Expression> projectMap) {
return repeat.getOutputs().stream()
.map(expr -> (NamedExpression) ExpressionUtils.replaceNameExpression(expr, projectMap))
.collect(Collectors.toList());
}
private List<Expression> nonVirtualGroupByExprs(LogicalAggregate<? extends Plan> agg) {
return agg.getGroupByExpressions().stream()
.filter(expr -> !(expr instanceof VirtualSlotReference))