database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](nereids)fix bug of select mv in nereids (#26235)

Browse Source 
* revert some change from pr26192
* disable some case for nereids
This commit is contained in:
starocean999
2023-11-03 20:28:55 +08:00
committed by GitHub
parent d367d9b585
commit 8c3e173553
9 changed files with 191 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java
View File

@ -77,7 +77,28 @@ import java.util.stream.Collectors;
* Base class for selecting materialized index rules.
*/
public abstract class AbstractSelectMaterializedIndexRule {
protected boolean shouldSelectIndex(LogicalOlapScan scan) {
protected boolean shouldSelectIndexWithAgg(LogicalOlapScan scan) {
switch (scan.getTable().getKeysType()) {
case AGG_KEYS:
case UNIQUE_KEYS:
case DUP_KEYS:
// SelectMaterializedIndexWithAggregate(R1) run before SelectMaterializedIndexWithoutAggregate(R2)
// if R1 selects baseIndex and preAggStatus is off
// we should give a chance to R2 to check if some prefix-index can be selected
// so if R1 selects baseIndex and preAggStatus is off, we keep scan's index unselected in order to
// let R2 to get a chance to do its work
// at last, after R1, the scan may be the 4 status
// 1. preAggStatus is ON and baseIndex is selected, it means select baseIndex is correct.
// 2. preAggStatus is ON and some other Index is selected, this is correct, too.
// 3. preAggStatus is OFF, no index is selected, it means R2 could get a chance to run
// so we check the preAggStatus and if some index is selected to make sure R1 can be run only once
return scan.getPreAggStatus().isOn() && !scan.isIndexSelected();
default:
return false;
}
}
protected boolean shouldSelectIndexWithoutAgg(LogicalOlapScan scan) {
switch (scan.getTable().getKeysType()) {
case AGG_KEYS:
case UNIQUE_KEYS:

164
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java
View File

@ -105,7 +105,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
return ImmutableList.of(
// only agg above scan
// Aggregate(Scan)
logicalAggregate(logicalOlapScan().when(this::shouldSelectIndex)).thenApply(ctx -> {
logicalAggregate(logicalOlapScan().when(this::shouldSelectIndexWithAgg)).thenApply(ctx -> {
LogicalAggregate<LogicalOlapScan> agg = ctx.root;
LogicalOlapScan scan = agg.child();
SelectResult result = select(
@ -116,11 +116,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
agg.getGroupByExpressions(),
new HashSet<>(agg.getExpressions()));
if (result.indexId == scan.getTable().getBaseIndexId()) {
return ctx.root;
}
LogicalOlapScan mvPlan = scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
if (result.exprRewriteMap.isEmpty()) {
@ -143,7 +139,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
// filter could push down scan.
// Aggregate(Filter(Scan))
logicalAggregate(logicalFilter(logicalOlapScan().when(this::shouldSelectIndex)))
logicalAggregate(logicalFilter(logicalOlapScan().when(this::shouldSelectIndexWithAgg)))
.thenApply(ctx -> {
LogicalAggregate<LogicalFilter<LogicalOlapScan>> agg = ctx.root;
LogicalFilter<LogicalOlapScan> filter = agg.child();
@ -166,12 +162,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
requiredExpr
);
if (result.indexId == scan.getTable().getBaseIndexId()) {
return ctx.root;
}
LogicalOlapScan mvPlan =
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
if (result.exprRewriteMap.isEmpty()) {
@ -199,7 +190,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
// column pruning or other projections such as alias, etc.
// Aggregate(Project(Scan))
logicalAggregate(logicalProject(logicalOlapScan().when(this::shouldSelectIndex)))
logicalAggregate(logicalProject(logicalOlapScan().when(this::shouldSelectIndexWithAgg)))
.thenApply(ctx -> {
LogicalAggregate<LogicalProject<LogicalOlapScan>> agg = ctx.root;
LogicalProject<LogicalOlapScan> project = agg.child();
@ -215,12 +206,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
collectRequireExprWithAggAndProject(agg.getExpressions(), project.getProjects())
);
if (result.indexId == scan.getTable().getBaseIndexId()) {
return ctx.root;
}
LogicalOlapScan mvPlan =
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
if (result.exprRewriteMap.isEmpty()) {
@ -254,7 +240,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
// filter could push down and project.
// Aggregate(Project(Filter(Scan)))
logicalAggregate(logicalProject(logicalFilter(logicalOlapScan()
.when(this::shouldSelectIndex)))).thenApply(ctx -> {
.when(this::shouldSelectIndexWithAgg)))).thenApply(ctx -> {
LogicalAggregate<LogicalProject<LogicalFilter<LogicalOlapScan>>> agg = ctx.root;
LogicalProject<LogicalFilter<LogicalOlapScan>> project = agg.child();
LogicalFilter<LogicalOlapScan> filter = project.child();
@ -277,12 +263,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
requiredExpr
);
if (result.indexId == scan.getTable().getBaseIndexId()) {
return ctx.root;
}
LogicalOlapScan mvPlan =
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
if (result.exprRewriteMap.isEmpty()) {
@ -317,7 +298,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
// filter can't push down
// Aggregate(Filter(Project(Scan)))
logicalAggregate(logicalFilter(logicalProject(logicalOlapScan()
.when(this::shouldSelectIndex)))).thenApply(ctx -> {
.when(this::shouldSelectIndexWithAgg)))).thenApply(ctx -> {
LogicalAggregate<LogicalFilter<LogicalProject<LogicalOlapScan>>> agg = ctx.root;
LogicalFilter<LogicalProject<LogicalOlapScan>> filter = agg.child();
LogicalProject<LogicalOlapScan> project = filter.child();
@ -338,12 +319,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
requiredExpr
);
if (result.indexId == scan.getTable().getBaseIndexId()) {
return ctx.root;
}
LogicalOlapScan mvPlan =
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
if (result.exprRewriteMap.isEmpty()) {
@ -377,52 +353,49 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
// only agg above scan
// Aggregate(Repeat(Scan))
logicalAggregate(logicalRepeat(logicalOlapScan().when(this::shouldSelectIndex))).thenApply(ctx -> {
LogicalAggregate<LogicalRepeat<LogicalOlapScan>> agg = ctx.root;
LogicalRepeat<LogicalOlapScan> repeat = agg.child();
LogicalOlapScan scan = repeat.child();
SelectResult result = select(
scan,
agg.getInputSlots(),
ImmutableSet.of(),
extractAggFunctionAndReplaceSlot(agg, Optional.empty()),
nonVirtualGroupByExprs(agg),
new HashSet<>(agg.getExpressions()));
logicalAggregate(
logicalRepeat(logicalOlapScan().when(this::shouldSelectIndexWithAgg))).thenApply(ctx -> {
LogicalAggregate<LogicalRepeat<LogicalOlapScan>> agg = ctx.root;
LogicalRepeat<LogicalOlapScan> repeat = agg.child();
LogicalOlapScan scan = repeat.child();
SelectResult result = select(
scan,
agg.getInputSlots(),
ImmutableSet.of(),
extractAggFunctionAndReplaceSlot(agg, Optional.empty()),
nonVirtualGroupByExprs(agg),
new HashSet<>(agg.getExpressions()));
if (result.indexId == scan.getTable().getBaseIndexId()) {
return ctx.root;
}
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
LogicalOlapScan mvPlan = scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
if (result.exprRewriteMap.isEmpty()) {
return new LogicalProject<>(
generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext).replace(
agg.withChildren(
repeat.withAggOutputAndChild(
generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), mvPlan)
), mvPlan));
} else {
return new LogicalProject<>(
generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(
agg.getGroupByExpressions(),
replaceAggOutput(
agg, Optional.empty(), Optional.empty(), result.exprRewriteMap),
agg.isNormalized(),
agg.getSourceRepeat(),
if (result.exprRewriteMap.isEmpty()) {
return new LogicalProject<>(
generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext).replace(
agg.withChildren(
repeat.withAggOutputAndChild(
generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), mvPlan)
), mvPlan));
}
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_SCAN),
} else {
return new LogicalProject<>(
generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(
agg.getGroupByExpressions(),
replaceAggOutput(
agg, Optional.empty(), Optional.empty(), result.exprRewriteMap),
agg.isNormalized(),
agg.getSourceRepeat(),
repeat.withAggOutputAndChild(
generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), mvPlan)
), mvPlan));
}
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_SCAN),
// filter could push down scan.
// Aggregate(Repeat(Filter(Scan)))
logicalAggregate(logicalRepeat(logicalFilter(logicalOlapScan().when(this::shouldSelectIndex))))
logicalAggregate(logicalRepeat(logicalFilter(logicalOlapScan().when(this::shouldSelectIndexWithAgg))))
.thenApply(ctx -> {
LogicalAggregate<LogicalRepeat<LogicalFilter<LogicalOlapScan>>> agg = ctx.root;
LogicalRepeat<LogicalFilter<LogicalOlapScan>> repeat = agg.child();
@ -446,12 +419,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
requiredExpr
);
if (result.indexId == scan.getTable().getBaseIndexId()) {
return ctx.root;
}
LogicalOlapScan mvPlan =
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
if (result.exprRewriteMap.isEmpty()) {
@ -485,7 +453,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
// column pruning or other projections such as alias, etc.
// Aggregate(Repeat(Project(Scan)))
logicalAggregate(logicalRepeat(logicalProject(logicalOlapScan().when(this::shouldSelectIndex))))
logicalAggregate(logicalRepeat(logicalProject(logicalOlapScan().when(this::shouldSelectIndexWithAgg))))
.thenApply(ctx -> {
LogicalAggregate<LogicalRepeat<LogicalProject<LogicalOlapScan>>> agg = ctx.root;
LogicalRepeat<LogicalProject<LogicalOlapScan>> repeat = agg.child();
@ -502,12 +470,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
collectRequireExprWithAggAndProject(agg.getExpressions(), project.getProjects())
);
if (result.indexId == scan.getTable().getBaseIndexId()) {
return ctx.root;
}
LogicalOlapScan mvPlan =
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
if (result.exprRewriteMap.isEmpty()) {
@ -546,7 +509,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
// filter could push down and project.
// Aggregate(Repeat(Project(Filter(Scan))))
logicalAggregate(logicalRepeat(logicalProject(logicalFilter(logicalOlapScan()
.when(this::shouldSelectIndex))))).thenApply(ctx -> {
.when(this::shouldSelectIndexWithAgg))))).thenApply(ctx -> {
LogicalAggregate<LogicalRepeat<LogicalProject
<LogicalFilter<LogicalOlapScan>>>> agg = ctx.root;
LogicalRepeat<LogicalProject<LogicalFilter<LogicalOlapScan>>> repeat = agg.child();
@ -571,12 +534,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
requiredExpr
);
if (result.indexId == scan.getTable().getBaseIndexId()) {
return ctx.root;
}
LogicalOlapScan mvPlan =
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
if (result.exprRewriteMap.isEmpty()) {
@ -618,7 +576,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
// filter can't push down
// Aggregate(Repeat(Filter(Project(Scan))))
logicalAggregate(logicalRepeat(logicalFilter(logicalProject(logicalOlapScan()
.when(this::shouldSelectIndex))))).thenApply(ctx -> {
.when(this::shouldSelectIndexWithAgg))))).thenApply(ctx -> {
LogicalAggregate<LogicalRepeat<LogicalFilter
<LogicalProject<LogicalOlapScan>>>> agg = ctx.root;
LogicalRepeat<LogicalFilter<LogicalProject<LogicalOlapScan>>> repeat = agg.child();
@ -641,12 +599,7 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
requiredExpr
);
if (result.indexId == scan.getTable().getBaseIndexId()) {
return ctx.root;
}
LogicalOlapScan mvPlan =
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
if (result.exprRewriteMap.isEmpty()) {
@ -687,6 +640,19 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
);
}
private static LogicalOlapScan createLogicalOlapScan(LogicalOlapScan scan, SelectResult result) {
LogicalOlapScan mvPlan;
if (result.preAggStatus.isOff()) {
// we only set preAggStatus and make index unselected to let SelectMaterializedIndexWithoutAggregate
// have a chance to run and select proper index
mvPlan = scan.withPreAggStatus(result.preAggStatus);
} else {
mvPlan =
scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId);
}
return mvPlan;
}
///////////////////////////////////////////////////////////////////////////
// Main entrance of select materialized index.
///////////////////////////////////////////////////////////////////////////

19
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithoutAggregate.java
View File

@ -24,6 +24,8 @@ import org.apache.doris.catalog.OlapTable;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory;
import org.apache.doris.nereids.rules.rewrite.mv.AbstractSelectMaterializedIndexRule.ReplaceExpressions;
import org.apache.doris.nereids.rules.rewrite.mv.AbstractSelectMaterializedIndexRule.SlotContext;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.PreAggStatus;
@ -59,7 +61,7 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater
return ImmutableList.of(
// project with pushdown filter.
// Project(Filter(Scan))
logicalProject(logicalFilter(logicalOlapScan().when(this::shouldSelectIndex)))
logicalProject(logicalFilter(logicalOlapScan().when(this::shouldSelectIndexWithoutAgg)))
.thenApply(ctx -> {
LogicalProject<LogicalFilter<LogicalOlapScan>> project = ctx.root;
LogicalFilter<LogicalOlapScan> filter = project.child();
@ -79,7 +81,7 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater
// project with filter that cannot be pushdown.
// Filter(Project(Scan))
logicalFilter(logicalProject(logicalOlapScan().when(this::shouldSelectIndex)))
logicalFilter(logicalProject(logicalOlapScan().when(this::shouldSelectIndexWithoutAgg)))
.thenApply(ctx -> {
LogicalFilter<LogicalProject<LogicalOlapScan>> filter = ctx.root;
LogicalProject<LogicalOlapScan> project = filter.child();
@ -98,13 +100,14 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater
// scan with filters could be pushdown.
// Filter(Scan)
logicalFilter(logicalOlapScan().when(this::shouldSelectIndex))
logicalFilter(logicalOlapScan().when(this::shouldSelectIndexWithoutAgg))
.thenApply(ctx -> {
LogicalFilter<LogicalOlapScan> filter = ctx.root;
LogicalOlapScan scan = filter.child();
LogicalOlapScan mvPlan = select(
scan, filter::getOutputSet, filter::getConjuncts,
new HashSet<>(filter.getExpressions()));
Stream.concat(filter.getExpressions().stream(),
filter.getOutputSet().stream()).collect(ImmutableSet.toImmutableSet()));
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
return new LogicalProject(
@ -116,7 +119,7 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater
// project and scan.
// Project(Scan)
logicalProject(logicalOlapScan().when(this::shouldSelectIndex))
logicalProject(logicalOlapScan().when(this::shouldSelectIndexWithoutAgg))
.thenApply(ctx -> {
LogicalProject<LogicalOlapScan> project = ctx.root;
LogicalOlapScan scan = project.child();
@ -135,7 +138,7 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater
// only scan.
logicalOlapScan()
.when(this::shouldSelectIndex)
.when(this::shouldSelectIndexWithoutAgg)
.thenApply(ctx -> {
LogicalOlapScan scan = ctx.root;
@ -196,7 +199,9 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater
// PreAggStatus could be enabled by pre-aggregation hint for agg-keys and unique-keys.
preAggStatus = PreAggStatus.on();
} else {
preAggStatus = PreAggStatus.off("No aggregate on scan.");
// if PreAggStatus is OFF, we use the message from SelectMaterializedIndexWithAggregate
preAggStatus = scan.getPreAggStatus().isOff() ? scan.getPreAggStatus()
: PreAggStatus.off("No aggregate on scan.");
}
if (table.getIndexIdToMeta().size() == 1) {
return scan.withMaterializedIndexSelected(preAggStatus, baseIndexId);

13
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
View File

@ -178,8 +178,12 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
@Override
public String toString() {
return Utils.toSqlString("LogicalOlapScan", "qualified", qualifiedName(), "indexName",
getSelectedMaterializedIndexName(), "selectedIndexId", selectedIndexId, "preAgg", preAggStatus);
return Utils.toSqlString("LogicalOlapScan",
"qualified", qualifiedName(),
"indexName", getSelectedMaterializedIndexName().orElse("<index_not_selected>"),
"selectedIndexId", selectedIndexId,
"preAgg", preAggStatus
);
}
@Override
@ -287,8 +291,9 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
}
@VisibleForTesting
public String getSelectedMaterializedIndexName() {
return ((OlapTable) table).getIndexNameById(selectedIndexId);
public Optional<String> getSelectedMaterializedIndexName() {
return indexSelected ? Optional.ofNullable(((OlapTable) table).getIndexNameById(selectedIndexId))
: Optional.empty();
}
@Override