database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Bug](materialized view) fix wrong match mv when mv have where clause (#21797)

Browse Source
This commit is contained in:
Pxl
2023-07-19 01:11:39 +08:00
committed by GitHub
parent 845cf94a7a
commit 0de94e857f
10 changed files with 244 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
View File

@ -268,6 +268,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
topDown(
new SelectMaterializedIndexWithAggregate(),
new SelectMaterializedIndexWithoutAggregate(),
new EliminateFilter(),
new PushdownFilterThroughProject(),
new MergeProjects(),
new PruneOlapScanTablet()

55
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java
View File

@ -20,6 +20,7 @@ package org.apache.doris.nereids.rules.rewrite.mv;
import org.apache.doris.analysis.CreateMaterializedViewStmt;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.trees.expressions.Alias;
@ -43,6 +44,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.HllHash;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunction;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBitmap;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBitmapWithCheck;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
@ -55,6 +57,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.planner.PlanNode;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@ -90,27 +93,33 @@ public abstract class AbstractSelectMaterializedIndexRule {
}
}
protected boolean containAllRequiredColumns(
MaterializedIndex index,
LogicalOlapScan scan,
Set<Slot> requiredScanOutput,
Set<? extends Expression> requiredExpr) {
protected boolean containAllRequiredColumns(MaterializedIndex index, LogicalOlapScan scan,
Set<Slot> requiredScanOutput, Set<? extends Expression> requiredExpr, Set<Expression> predicateExpr) {
OlapTable table = scan.getTable();
MaterializedIndexMeta meta = table.getIndexMetaByIndexId(index.getId());
Set<String> predicateExprSql = predicateExpr.stream().map(e -> e.toSql()).collect(Collectors.toSet());
Set<String> indexConjuncts = PlanNode.splitAndCompoundPredicateToConjuncts(meta.getWhereClause()).stream()
.map(e -> new NereidsParser().parseExpression(e.toSql()).toSql()).collect(Collectors.toSet());
Set<String> commonConjuncts = indexConjuncts.stream().filter(e -> predicateExprSql.contains(e))
.collect(Collectors.toSet());
if (commonConjuncts.size() != indexConjuncts.size()) {
return false;
}
Set<String> requiredMvColumnNames = requiredScanOutput.stream()
.map(s -> normalizeName(Column.getNameWithoutMvPrefix(s.getName())))
.collect(Collectors.toCollection(() -> new TreeSet<String>(String.CASE_INSENSITIVE_ORDER)));
Set<String> mvColNames = table.getSchemaByIndexId(index.getId(), true).stream()
Set<String> mvColNames = meta.getSchema().stream()
.map(c -> normalizeName(parseMvColumnToSql(c.getNameWithoutMvPrefix())))
.collect(Collectors.toCollection(() -> new TreeSet<String>(String.CASE_INSENSITIVE_ORDER)));
mvColNames.addAll(indexConjuncts);
return mvColNames.containsAll(requiredMvColumnNames)
|| requiredExpr.stream()
.map(AbstractSelectMaterializedIndexRule::removeCastAndAlias)
.filter(e -> !containsAllColumn(e, mvColNames))
.collect(Collectors.toSet()).isEmpty();
&& (indexConjuncts.isEmpty() || commonConjuncts.size() == predicateExprSql.size())
|| requiredExpr.stream().map(AbstractSelectMaterializedIndexRule::removeCastAndAlias)
.filter(e -> !containsAllColumn(e, mvColNames)).collect(Collectors.toSet()).isEmpty();
}
public static String parseMvColumnToSql(String mvName) {
@ -386,7 +395,7 @@ public abstract class AbstractSelectMaterializedIndexRule {
Map<Slot, Slot> baseSlotToMvSlot = new HashMap<>();
Map<String, Slot> mvNameToMvSlot = new HashMap<>();
if (mvPlan.getSelectedIndexId() == mvPlan.getTable().getBaseIndexId()) {
return new SlotContext(baseSlotToMvSlot, mvNameToMvSlot);
return new SlotContext(baseSlotToMvSlot, mvNameToMvSlot, new TreeSet<Expression>());
}
for (Slot mvSlot : mvPlan.getOutputByIndex(mvPlan.getSelectedIndexId())) {
boolean isPushed = false;
@ -411,7 +420,12 @@ public abstract class AbstractSelectMaterializedIndexRule {
mvNameToMvSlot.put(normalizeName(mvSlot.getName()), mvSlot);
}
}
return new SlotContext(baseSlotToMvSlot, mvNameToMvSlot);
OlapTable table = mvPlan.getTable();
MaterializedIndexMeta meta = table.getIndexMetaByIndexId(mvPlan.getSelectedIndexId());
return new SlotContext(baseSlotToMvSlot, mvNameToMvSlot,
PlanNode.splitAndCompoundPredicateToConjuncts(meta.getWhereClause()).stream()
.map(e -> new NereidsParser().parseExpression(e.toSql())).collect(Collectors.toSet()));
}
/** SlotContext */
@ -422,9 +436,13 @@ public abstract class AbstractSelectMaterializedIndexRule {
// selected mv Slot name to mv Slot, we must use ImmutableSortedMap because column name could be uppercase
public final ImmutableSortedMap<String, Slot> mvNameToMvSlot;
public SlotContext(Map<Slot, Slot> baseSlotToMvSlot, Map<String, Slot> mvNameToMvSlot) {
public final ImmutableSet<Expression> trueExprs;
public SlotContext(Map<Slot, Slot> baseSlotToMvSlot, Map<String, Slot> mvNameToMvSlot,
Set<Expression> trueExprs) {
this.baseSlotToMvSlot = ImmutableMap.copyOf(baseSlotToMvSlot);
this.mvNameToMvSlot = ImmutableSortedMap.copyOf(mvNameToMvSlot, String.CASE_INSENSITIVE_ORDER);
this.trueExprs = ImmutableSet.copyOf(trueExprs);
}
}
@ -520,9 +538,12 @@ public abstract class AbstractSelectMaterializedIndexRule {
// selected mv Slot name to mv Slot, we must use ImmutableSortedMap because column name could be uppercase
private final ImmutableSortedMap<String, Slot> mvNameToMvSlot;
private final ImmutableSet<String> trueExprs;
public ReplaceExpressionWithMvColumn(SlotContext slotContext) {
this.baseSlotToMvSlot = ImmutableMap.copyOf(slotContext.baseSlotToMvSlot);
this.mvNameToMvSlot = ImmutableSortedMap.copyOf(slotContext.mvNameToMvSlot, String.CASE_INSENSITIVE_ORDER);
this.trueExprs = slotContext.trueExprs.stream().map(e -> e.toSql()).collect(ImmutableSet.toImmutableSet());
}
public Expression replace(Expression expression) {
@ -533,9 +554,11 @@ public abstract class AbstractSelectMaterializedIndexRule {
public Expression visit(Expression expr, Void context) {
if (notUseMv() || org.apache.doris.analysis.CreateMaterializedViewStmt.isMVColumn(expr.toSql())) {
return expr;
} else if (trueExprs.contains(expr.toSql())) {
return BooleanLiteral.TRUE;
} else if (checkExprIsMvColumn(expr)) {
return mvNameToMvSlot.get(
org.apache.doris.analysis.CreateMaterializedViewStmt.mvColumnBuilder(expr.toSql()));
return mvNameToMvSlot
.get(org.apache.doris.analysis.CreateMaterializedViewStmt.mvColumnBuilder(expr.toSql()));
} else {
expr = super.visit(expr, context);
return expr;

18
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java
View File

@ -711,12 +711,11 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
List<MaterializedIndex> haveAllRequiredColumns = Streams.concat(
candidatesWithoutRewriting.stream()
.filter(index -> containAllRequiredColumns(
index, scan, nonVirtualRequiredScanOutput, requiredExpr)),
candidatesWithRewriting
.stream()
.filter(index -> containAllRequiredColumns(index, scan, nonVirtualRequiredScanOutput,
requiredExpr, predicates)),
candidatesWithRewriting.stream()
.filter(aggRewriteResult -> containAllRequiredColumns(aggRewriteResult.index, scan,
aggRewriteResult.requiredScanOutput, requiredExpr))
aggRewriteResult.requiredScanOutput, requiredExpr, predicates))
.map(aggRewriteResult -> aggRewriteResult.index)
).collect(Collectors.toList());
@ -757,11 +756,10 @@ public class SelectMaterializedIndexWithAggregate extends AbstractSelectMaterial
return baseIndexSelectResult;
} else {
List<MaterializedIndex> rollupsWithAllRequiredCols =
Stream.concat(candidatesWithoutRewriting.stream(),
indexesGroupByIsBaseOrNot.get(true).stream())
.filter(index -> containAllRequiredColumns(
index, scan, nonVirtualRequiredScanOutput, requiredExpr))
.collect(Collectors.toList());
Stream.concat(candidatesWithoutRewriting.stream(), indexesGroupByIsBaseOrNot.get(true).stream())
.filter(index -> containAllRequiredColumns(index, scan, nonVirtualRequiredScanOutput,
requiredExpr, predicates))
.collect(Collectors.toList());
long selectedIndex = selectBestIndex(rollupsWithAllRequiredCols, scan, predicates);
if (selectedIndex == scan.getTable().getBaseIndexId()) {

14
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithoutAggregate.java
View File

@ -183,11 +183,10 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater
}
if (scan.getTable().isDupKeysOrMergeOnWrite()) {
// Set pre-aggregation to `on` to keep consistency with legacy logic.
List<MaterializedIndex> candidates = scan.getTable().getVisibleIndex().stream()
.filter(index -> index.getId() != baseIndexId)
.filter(index -> !indexHasAggregate(index, scan))
.filter(index -> containAllRequiredColumns(index, scan,
requiredScanOutputSupplier.get(), requiredExpr))
List<MaterializedIndex> candidates = scan
.getTable().getVisibleIndex().stream().filter(index -> index.getId() != baseIndexId)
.filter(index -> !indexHasAggregate(index, scan)).filter(index -> containAllRequiredColumns(index,
scan, requiredScanOutputSupplier.get(), requiredExpr, predicatesSupplier.get()))
.collect(Collectors.toList());
long bestIndex = selectBestIndex(candidates, scan, predicatesSupplier.get());
return scan.withMaterializedIndexSelected(PreAggStatus.on(), bestIndex);
@ -207,9 +206,8 @@ public class SelectMaterializedIndexWithoutAggregate extends AbstractSelectMater
// So only base index and indexes that have all the keys could be used.
List<MaterializedIndex> candidates = table.getVisibleIndex().stream()
.filter(index -> table.getKeyColumnsByIndexId(index.getId()).size() == baseIndexKeySize)
.filter(index -> containAllRequiredColumns(
index, scan, requiredScanOutputSupplier.get(),
predicatesSupplier.get()))
.filter(index -> containAllRequiredColumns(index, scan, requiredScanOutputSupplier.get(),
requiredExpr, predicatesSupplier.get()))
.collect(Collectors.toList());
if (candidates.size() == 1) {

4
fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
View File

@ -100,7 +100,6 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@ -359,8 +358,7 @@ public class OlapScanNode extends ScanNode {
return;
}
Expr vconjunct = convertConjunctsToAndCompoundPredicate(conjuncts).replaceSubPredicate(whereExpr);
conjuncts = splitAndCompoundPredicateToConjuncts(vconjunct).stream().filter(Objects::nonNull)
.collect(Collectors.toList());
conjuncts = splitAndCompoundPredicateToConjuncts(vconjunct).stream().collect(Collectors.toList());
}
/**

2
fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
View File

@ -439,7 +439,7 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
conjuncts.addAll(splitAndCompoundPredicateToConjuncts(vconjunct.getChild(1)));
}
}
if (conjuncts.isEmpty()) {
if (vconjunct != null && conjuncts.isEmpty()) {
conjuncts.add(vconjunct);
}
return conjuncts;