database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Fix](nereids) Fix cte rewrite by mv failure and predicates compensation by mistake (#29820)

Browse Source 
Fix cte rewrite by mv wrongly when query has scalar aggregate but view no
For example as following, it should not be rewritten by materialized view successfully

// materialzied view define
def mv20_1 = """
select
l_shipmode,
l_shipinstruct,
sum(l_extendedprice),
count()
from lineitem
left join
orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY
group by
l_shipmode,
l_shipinstruct;
"""
// query sql
def query20_1 =
"""
select
sum(l_extendedprice),
count()
from lineitem
left join
orders
on lineitem.L_ORDERKEY = orders.O_ORDERKEY
"""

Fix predicates compensation by mistake
For example as following, it can return right result, but it's wrong earlier.

// materialzied view define
def mv7_1 = """
select l_shipdate, o_orderdate, l_partkey, l_suppkey
from lineitem
left join orders
on lineitem.l_orderkey = orders.o_orderkey
where l_shipdate = '2023-12-08' and o_orderdate = '2023-12-08';
"""
// query sql
def query7_1 = """
select l_shipdate, o_orderdate, l_partkey, l_suppkey
from (select * from lineitem where l_shipdate = '2023-10-17' ) t1
left join orders
on t1.l_orderkey = orders.o_orderkey;
"""

and optimize some code usage and add more comment for method
This commit is contained in:
seawinde
2024-01-12 16:30:35 +08:00
committed by yiguolei
parent e417128fb9
commit d47adbb81f
22 changed files with 716 additions and 256 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
View File

@ -48,8 +48,6 @@ import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.Collection;
@ -68,8 +66,6 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
protected static final Multimap<Expression, Expression>
AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP = ArrayListMultimap.create();
protected final String currentClassName = this.getClass().getSimpleName();
private final Logger logger = LogManager.getLogger(this.getClass());
static {
// support count distinct roll up
@ -142,6 +138,18 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
materializationContext.getMvExprToMvScanExprMapping(),
queryToViewSlotMapping)));
}
// if view is scalar aggregate but query is not. Or if query is scalar aggregate but view is not
// Should not rewrite
if (queryTopPlanAndAggPair.value().getGroupByExpressions().isEmpty()
|| viewTopPlanAndAggPair.value().getGroupByExpressions().isEmpty()) {
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
Pair.of("only one the of query or view is scalar aggregate and "
+ "can not rewrite expression meanwhile",
String.format("query aggregate = %s,\n view aggregate = %s,\n",
queryTopPlanAndAggPair.value().treeString(),
viewTopPlanAndAggPair.value().treeString())));
return null;
}
// try to roll up.
// split the query top plan expressions to group expressions and functions, if can not, bail out.
Pair<Set<? extends Expression>, Set<? extends Expression>> queryGroupAndFunctionPair
@ -268,15 +276,15 @@ public abstract class AbstractMaterializedViewAggregateRule extends AbstractMate
Plan viewTopPlan = viewTopPlanAndAggPair.key();
LogicalAggregate<Plan> queryAggregate = queryTopPlanAndAggPair.value();
LogicalAggregate<Plan> viewAggregate = viewTopPlanAndAggPair.value();
List<? extends Expression> queryGroupShuttledExpression = ExpressionUtils.shuttleExpressionWithLineage(
queryAggregate.getGroupByExpressions(), queryTopPlan);
List<? extends Expression> viewGroupShuttledExpression = ExpressionUtils.shuttleExpressionWithLineage(
Set<? extends Expression> queryGroupShuttledExpression = new HashSet<>(
ExpressionUtils.shuttleExpressionWithLineage(
queryAggregate.getGroupByExpressions(), queryTopPlan));
Set<? extends Expression> viewGroupShuttledExpressionQueryBased = ExpressionUtils.shuttleExpressionWithLineage(
viewAggregate.getGroupByExpressions(), viewTopPlan)
.stream()
.map(expr -> ExpressionUtils.replace(expr, viewToQurySlotMapping.toSlotReferenceMap()))
.collect(Collectors.toList());
return queryAggregate.getGroupByExpressions().size() == viewAggregate.getGroupByExpressions().size()
&& queryGroupShuttledExpression.equals(viewGroupShuttledExpression);
.collect(Collectors.toSet());
return queryGroupShuttledExpression.equals(viewGroupShuttledExpressionQueryBased);
}
/**

215
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View File

@ -31,16 +31,13 @@ import org.apache.doris.nereids.jobs.executor.Rewriter;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.rules.exploration.ExplorationRuleFactory;
import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate;
import org.apache.doris.nereids.rules.exploration.mv.mapping.EquivalenceClassSetMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
@ -61,7 +58,6 @@ import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@ -85,7 +81,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
if (materializationContexts.isEmpty()) {
return rewriteResults;
}
List<StructInfo> queryStructInfos = extractStructInfo(queryPlan, cascadesContext);
List<StructInfo> queryStructInfos = MaterializedViewUtils.extractStructInfo(queryPlan, cascadesContext);
// TODO Just Check query queryPlan firstly, support multi later.
StructInfo queryStructInfo = queryStructInfos.get(0);
if (!checkPattern(queryStructInfo)) {
@ -99,7 +95,8 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
if (checkIfRewritten(queryPlan, materializationContext)) {
continue;
}
List<StructInfo> viewStructInfos = extractStructInfo(materializationContext.getMvPlan(), cascadesContext);
List<StructInfo> viewStructInfos = MaterializedViewUtils.extractStructInfo(
materializationContext.getMvPlan(), cascadesContext);
if (viewStructInfos.size() > 1) {
// view struct info should only have one
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
@ -145,16 +142,10 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
comparisonResult.getErrorMessage()));
continue;
}
// TODO: Use set of list? And consider view expr
List<Expression> pulledUpExpressions = ImmutableList.copyOf(comparisonResult.getQueryExpressions());
// set pulled up expression to queryStructInfo predicates and update related predicates
if (!pulledUpExpressions.isEmpty()) {
queryStructInfo.addPredicates(pulledUpExpressions);
}
SplitPredicate compensatePredicates = predicatesCompensate(queryStructInfo, viewStructInfo,
queryToViewSlotMapping, comparisonResult, cascadesContext);
// Can not compensate, bail out
if (compensatePredicates.isEmpty()) {
if (compensatePredicates.isInvalid()) {
materializationContext.recordFailReason(queryStructInfo.getOriginalPlanId(),
Pair.of("Predicate compensate fail",
String.format("query predicates = %s,\n query equivalenceClass = %s, \n"
@ -222,6 +213,9 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
return rewriteResults;
}
/**
* Check the logical properties of rewritten plan by mv is the same with source plan
*/
protected boolean checkOutput(Plan sourcePlan, Plan rewrittenPlan, MaterializationContext materializationContext) {
if (sourcePlan.getGroupExpression().isPresent() && !rewrittenPlan.getLogicalProperties()
.equals(sourcePlan.getGroupExpression().get().getOwnerGroup().getLogicalProperties())) {
@ -238,7 +232,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
}
/**
* Partition will be pruned in query then add the record the partitions to select partitions on
* Partition will be pruned in query then add the pruned partitions to select partitions field of
* catalog relation.
* Maybe only just some partitions is valid in materialized view, so we should check if the mv can
* offer the partitions which query used or not.
@ -285,10 +279,23 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
* Use target expression to represent the source expression. Visit the source expression,
* try to replace the source expression with target expression in targetExpressionMapping, if found then
* replace the source expression by target expression mapping value.
* Note: make the target expression map key to source based according to targetExpressionNeedSourceBased,
* if targetExpressionNeedSourceBased is true, we should make it source based.
* the key expression in targetExpressionMapping should be shuttled. with the method
* ExpressionUtils.shuttleExpressionWithLineage.
*
* @param sourceExpressionsToWrite the source expression to write by target expression
* @param sourcePlan the source plan witch the source expression belong to
* @param targetExpressionMapping target expression mapping, if finding the expression in key set of the mapping
* then use the corresponding value of mapping to replace it
* @param targetExpressionNeedSourceBased if targetExpressionNeedSourceBased is true,
* we should make the target expression map key to source based,
* Note: the key expression in targetExpressionMapping should be shuttled. with the method
* ExpressionUtils.shuttleExpressionWithLineage.
* example as following:
* source target
* project(slot 1, 2) project(slot 3, 2, 1)
* scan(table) scan(table)
* then
* transform source to:
* project(slot 2, 1)
* target
*/
protected List<Expression> rewriteExpression(List<? extends Expression> sourceExpressionsToWrite, Plan sourcePlan,
ExpressionMapping targetExpressionMapping, SlotMapping sourceToTargetMapping,
@ -296,15 +303,6 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
// Firstly, rewrite the target expression using source with inverse mapping
// then try to use the target expression to represent the query. if any of source expressions
// can not be represented by target expressions, return null.
//
// example as following:
// source target
// project(slot 1, 2) project(slot 3, 2, 1)
// scan(table) scan(table)
//
// transform source to:
// project(slot 2, 1)
// target
// generate target to target replacement expression mapping, and change target expression to source based
List<? extends Expression> sourceShuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage(
sourceExpressionsToWrite, sourcePlan);
@ -348,6 +346,9 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
return rewrittenExpressions;
}
/**
* Rewrite single expression, the logic is the same with above
*/
protected Expression rewriteExpression(Expression sourceExpressionsToWrite, Plan sourcePlan,
ExpressionMapping targetExpressionMapping, SlotMapping sourceToTargetMapping,
boolean targetExpressionNeedSourceBased) {
@ -374,27 +375,43 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
ComparisonResult comparisonResult,
CascadesContext cascadesContext
) {
// TODO: Use set of list? And consider view expr
List<Expression> queryPulledUpExpressions = ImmutableList.copyOf(comparisonResult.getQueryExpressions());
// set pulled up expression to queryStructInfo predicates and update related predicates
if (!queryPulledUpExpressions.isEmpty()) {
queryStructInfo.addPredicates(queryPulledUpExpressions);
}
List<Expression> viewPulledUpExpressions = ImmutableList.copyOf(comparisonResult.getViewExpressions());
// set pulled up expression to viewStructInfo predicates and update related predicates
if (!viewPulledUpExpressions.isEmpty()) {
viewStructInfo.addPredicates(viewPulledUpExpressions);
}
// viewEquivalenceClass to query based
SlotMapping viewToQuerySlotMapping = queryToViewSlotMapping.inverse();
final Set<Expression> equalCompensateConjunctions = compensateEquivalence(
// equal predicate compensate
final Set<Expression> equalCompensateConjunctions = Predicates.compensateEquivalence(
queryStructInfo,
viewStructInfo,
viewToQuerySlotMapping,
comparisonResult);
// range compensate
final Set<Expression> rangeCompensatePredicates = compensateRangePredicate(
final Set<Expression> rangeCompensatePredicates = Predicates.compensateRangePredicate(
queryStructInfo,
viewStructInfo,
viewToQuerySlotMapping,
comparisonResult);
// residual compensate
final Set<Expression> residualCompensatePredicates = compensateResidualPredicate(
final Set<Expression> residualCompensatePredicates = Predicates.compensateResidualPredicate(
queryStructInfo,
viewStructInfo,
viewToQuerySlotMapping,
comparisonResult);
// if the join type in query and mv plan is different, we should check and add filter on mv to make
// the mv join type is accord with query
if (equalCompensateConjunctions == null || rangeCompensatePredicates == null
|| residualCompensatePredicates == null) {
return SplitPredicate.INVALID_INSTANCE;
}
// if the join type in query and mv plan is different, we should check query is have the
// filters which rejects null
Set<Set<Slot>> requireNoNullableViewSlot = comparisonResult.getViewNoNullableSlot();
// check query is use the null reject slot which view comparison need
if (!requireNoNullableViewSlot.isEmpty()) {
@ -403,7 +420,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
cascadesContext);
if (nullRejectPredicates.isEmpty() || queryPulledUpPredicates.containsAll(nullRejectPredicates)) {
// query has not null reject predicates, so return
return SplitPredicate.invalid();
return SplitPredicate.INVALID_INSTANCE;
}
Set<Expression> queryUsedNeedRejectNullSlotsViewBased = nullRejectPredicates.stream()
.map(expression -> TypeUtils.isNotNull(expression).orElse(null))
@ -413,122 +430,17 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
.collect(Collectors.toSet());
if (requireNoNullableViewSlot.stream().anyMatch(
set -> Sets.intersection(set, queryUsedNeedRejectNullSlotsViewBased).isEmpty())) {
return SplitPredicate.invalid();
return SplitPredicate.INVALID_INSTANCE;
}
}
return SplitPredicate.of(ExpressionUtils.and(equalCompensateConjunctions),
rangeCompensatePredicates.isEmpty() ? BooleanLiteral.of(true)
return SplitPredicate.of(equalCompensateConjunctions.isEmpty() ? BooleanLiteral.TRUE
: ExpressionUtils.and(equalCompensateConjunctions),
rangeCompensatePredicates.isEmpty() ? BooleanLiteral.TRUE
: ExpressionUtils.and(rangeCompensatePredicates),
residualCompensatePredicates.isEmpty() ? BooleanLiteral.of(true)
residualCompensatePredicates.isEmpty() ? BooleanLiteral.TRUE
: ExpressionUtils.and(residualCompensatePredicates));
}
protected Set<Expression> compensateEquivalence(StructInfo queryStructInfo,
StructInfo viewStructInfo,
SlotMapping viewToQuerySlotMapping,
ComparisonResult comparisonResult) {
EquivalenceClass queryEquivalenceClass = queryStructInfo.getEquivalenceClass();
EquivalenceClass viewEquivalenceClass = viewStructInfo.getEquivalenceClass();
Map<SlotReference, SlotReference> viewToQuerySlotMap = viewToQuerySlotMapping.toSlotReferenceMap();
EquivalenceClass viewEquivalenceClassQueryBased = viewEquivalenceClass.permute(viewToQuerySlotMap);
if (viewEquivalenceClassQueryBased == null) {
return ImmutableSet.of();
}
final Set<Expression> equalCompensateConjunctions = new HashSet<>();
if (queryEquivalenceClass.isEmpty() && viewEquivalenceClass.isEmpty()) {
equalCompensateConjunctions.add(BooleanLiteral.of(true));
}
if (queryEquivalenceClass.isEmpty()
&& !viewEquivalenceClass.isEmpty()) {
return ImmutableSet.of();
}
EquivalenceClassSetMapping queryToViewEquivalenceMapping =
EquivalenceClassSetMapping.generate(queryEquivalenceClass, viewEquivalenceClassQueryBased);
// can not map all target equivalence class, can not compensate
if (queryToViewEquivalenceMapping.getEquivalenceClassSetMap().size()
< viewEquivalenceClass.getEquivalenceSetList().size()) {
return ImmutableSet.of();
}
// do equal compensate
Set<Set<SlotReference>> mappedQueryEquivalenceSet =
queryToViewEquivalenceMapping.getEquivalenceClassSetMap().keySet();
queryEquivalenceClass.getEquivalenceSetList().forEach(
queryEquivalenceSet -> {
// compensate the equivalence in query but not in view
if (!mappedQueryEquivalenceSet.contains(queryEquivalenceSet)) {
Iterator<SlotReference> iterator = queryEquivalenceSet.iterator();
SlotReference first = iterator.next();
while (iterator.hasNext()) {
Expression equals = new EqualTo(first, iterator.next());
equalCompensateConjunctions.add(equals);
}
} else {
// compensate the equivalence both in query and view, but query has more equivalence
Set<SlotReference> viewEquivalenceSet =
queryToViewEquivalenceMapping.getEquivalenceClassSetMap().get(queryEquivalenceSet);
Set<SlotReference> copiedQueryEquivalenceSet = new HashSet<>(queryEquivalenceSet);
copiedQueryEquivalenceSet.removeAll(viewEquivalenceSet);
SlotReference first = viewEquivalenceSet.iterator().next();
for (SlotReference slotReference : copiedQueryEquivalenceSet) {
Expression equals = new EqualTo(first, slotReference);
equalCompensateConjunctions.add(equals);
}
}
}
);
return equalCompensateConjunctions;
}
protected Set<Expression> compensateResidualPredicate(StructInfo queryStructInfo,
StructInfo viewStructInfo,
SlotMapping viewToQuerySlotMapping,
ComparisonResult comparisonResult) {
SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate();
SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate();
Expression queryResidualPredicate = querySplitPredicate.getResidualPredicate();
Expression viewResidualPredicate = viewSplitPredicate.getResidualPredicate();
Expression viewResidualPredicateQueryBased =
ExpressionUtils.replace(viewResidualPredicate, viewToQuerySlotMapping.toSlotReferenceMap());
Set<Expression> queryResidualSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(queryResidualPredicate));
Set<Expression> viewResidualQueryBasedSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(viewResidualPredicateQueryBased));
// query residual predicate can not contain all view residual predicate when view have residual predicate,
// bail out
if (!viewResidualPredicateQueryBased.equals(BooleanLiteral.TRUE)
&& !queryResidualSet.containsAll(viewResidualQueryBasedSet)) {
return ImmutableSet.of();
}
queryResidualSet.removeAll(viewResidualQueryBasedSet);
return queryResidualSet;
}
protected Set<Expression> compensateRangePredicate(StructInfo queryStructInfo,
StructInfo viewStructInfo,
SlotMapping viewToQuerySlotMapping,
ComparisonResult comparisonResult) {
// TODO range predicates and residual predicates compensate, Simplify implementation.
SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate();
SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate();
Expression queryRangePredicate = querySplitPredicate.getRangePredicate();
Expression viewRangePredicate = viewSplitPredicate.getRangePredicate();
Expression viewRangePredicateQueryBased =
ExpressionUtils.replace(viewRangePredicate, viewToQuerySlotMapping.toSlotReferenceMap());
Set<Expression> queryRangeSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(queryRangePredicate));
Set<Expression> viewRangeQueryBasedSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(viewRangePredicateQueryBased));
// query range predicate can not contain all view range predicate when view have range predicate, bail out
if (!viewRangePredicateQueryBased.equals(BooleanLiteral.TRUE)
&& !queryRangeSet.containsAll(viewRangeQueryBasedSet)) {
return ImmutableSet.of();
}
queryRangeSet.removeAll(viewRangeQueryBasedSet);
return queryRangeSet;
}
/**
* Decide the match mode
*
@ -554,23 +466,6 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
return MatchMode.NOT_MATCH;
}
/**
* Extract struct info from plan, support to get struct info from logical plan or plan in group.
*/
public static List<StructInfo> extractStructInfo(Plan plan, CascadesContext cascadesContext) {
if (plan.getGroupExpression().isPresent() && !plan.getGroupExpression().get().getOwnerGroup().getStructInfos()
.isEmpty()) {
return plan.getGroupExpression().get().getOwnerGroup().getStructInfos();
} else {
// build struct info and add them to current group
List<StructInfo> structInfos = StructInfo.of(plan);
if (plan.getGroupExpression().isPresent()) {
plan.getGroupExpression().get().getOwnerGroup().addStructInfo(structInfos);
}
return structInfos;
}
}
/**
* Check the pattern of query or materializedView is supported or not.
*/

8
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java
View File

@ -78,10 +78,11 @@ public class LogicalCompatibilityContext {
}
/**
* generate logical compatibility context
* Generate logical compatibility context,
* this make expression mapping between query and view by relation and the slot in relation mapping
*/
public static LogicalCompatibilityContext from(RelationMapping relationMapping,
SlotMapping slotMapping,
SlotMapping queryToViewSlotMapping,
StructInfo queryStructInfo,
StructInfo viewStructInfo) {
// init node mapping
@ -101,7 +102,8 @@ public class LogicalCompatibilityContext {
}
}
// init expression mapping
Map<SlotReference, SlotReference> viewToQuerySlotMapping = slotMapping.inverse().toSlotReferenceMap();
Map<SlotReference, SlotReference> viewToQuerySlotMapping = queryToViewSlotMapping.inverse()
.toSlotReferenceMap();
Map<Expression, Expression> queryShuttledExprToExprMap =
queryStructInfo.getShuttledHashConjunctsToConjunctsMap();
Map<Expression, Expression> viewShuttledExprToExprMap =

18
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
View File

@ -23,6 +23,7 @@ import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.mtmv.BaseTableInfo;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
@ -124,6 +125,23 @@ public class MaterializedViewUtils {
return analyzedPlan.accept(TableQueryOperatorChecker.INSTANCE, null);
}
/**
* Extract struct info from plan, support to get struct info from logical plan or plan in group.
*/
public static List<StructInfo> extractStructInfo(Plan plan, CascadesContext cascadesContext) {
if (plan.getGroupExpression().isPresent() && !plan.getGroupExpression().get().getOwnerGroup().getStructInfos()
.isEmpty()) {
return plan.getGroupExpression().get().getOwnerGroup().getStructInfos();
} else {
// build struct info and add them to current group
List<StructInfo> structInfos = StructInfo.of(plan);
if (plan.getGroupExpression().isPresent()) {
plan.getGroupExpression().get().getOwnerGroup().addStructInfo(structInfos);
}
return structInfos;
}
}
private static final class TableQueryOperatorChecker extends DefaultPlanVisitor<Boolean, Void> {
public static final TableQueryOperatorChecker INSTANCE = new TableQueryOperatorChecker();

178
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java
View File

@ -17,15 +17,23 @@
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.rules.exploration.mv.mapping.EquivalenceClassSetMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.Utils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@ -72,6 +80,127 @@ public class Predicates {
return predicatesSplit.getSplitPredicate();
}
/**
* compensate equivalence predicates
*/
public static Set<Expression> compensateEquivalence(StructInfo queryStructInfo,
StructInfo viewStructInfo,
SlotMapping viewToQuerySlotMapping,
ComparisonResult comparisonResult) {
EquivalenceClass queryEquivalenceClass = queryStructInfo.getEquivalenceClass();
EquivalenceClass viewEquivalenceClass = viewStructInfo.getEquivalenceClass();
Map<SlotReference, SlotReference> viewToQuerySlotMap = viewToQuerySlotMapping.toSlotReferenceMap();
EquivalenceClass viewEquivalenceClassQueryBased = viewEquivalenceClass.permute(viewToQuerySlotMap);
if (viewEquivalenceClassQueryBased == null) {
return null;
}
final Set<Expression> equalCompensateConjunctions = new HashSet<>();
if (queryEquivalenceClass.isEmpty() && viewEquivalenceClass.isEmpty()) {
equalCompensateConjunctions.add(BooleanLiteral.TRUE);
}
if (queryEquivalenceClass.isEmpty()
&& !viewEquivalenceClass.isEmpty()) {
return null;
}
EquivalenceClassSetMapping queryToViewEquivalenceMapping =
EquivalenceClassSetMapping.generate(queryEquivalenceClass, viewEquivalenceClassQueryBased);
// can not map all target equivalence class, can not compensate
if (queryToViewEquivalenceMapping.getEquivalenceClassSetMap().size()
< viewEquivalenceClass.getEquivalenceSetList().size()) {
return null;
}
// do equal compensate
Set<Set<SlotReference>> mappedQueryEquivalenceSet =
queryToViewEquivalenceMapping.getEquivalenceClassSetMap().keySet();
queryEquivalenceClass.getEquivalenceSetList().forEach(
queryEquivalenceSet -> {
// compensate the equivalence in query but not in view
if (!mappedQueryEquivalenceSet.contains(queryEquivalenceSet)) {
Iterator<SlotReference> iterator = queryEquivalenceSet.iterator();
SlotReference first = iterator.next();
while (iterator.hasNext()) {
Expression equals = new EqualTo(first, iterator.next());
equalCompensateConjunctions.add(equals);
}
} else {
// compensate the equivalence both in query and view, but query has more equivalence
Set<SlotReference> viewEquivalenceSet =
queryToViewEquivalenceMapping.getEquivalenceClassSetMap().get(queryEquivalenceSet);
Set<SlotReference> copiedQueryEquivalenceSet = new HashSet<>(queryEquivalenceSet);
copiedQueryEquivalenceSet.removeAll(viewEquivalenceSet);
SlotReference first = viewEquivalenceSet.iterator().next();
for (SlotReference slotReference : copiedQueryEquivalenceSet) {
Expression equals = new EqualTo(first, slotReference);
equalCompensateConjunctions.add(equals);
}
}
}
);
return equalCompensateConjunctions;
}
/**
* compensate range predicates
*/
public static Set<Expression> compensateRangePredicate(StructInfo queryStructInfo,
StructInfo viewStructInfo,
SlotMapping viewToQuerySlotMapping,
ComparisonResult comparisonResult) {
// TODO Range predicates compensate, simplify implementation currently.
SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate();
SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate();
Expression queryRangePredicate = querySplitPredicate.getRangePredicate();
Expression viewRangePredicate = viewSplitPredicate.getRangePredicate();
Expression viewRangePredicateQueryBased =
ExpressionUtils.replace(viewRangePredicate, viewToQuerySlotMapping.toSlotReferenceMap());
Set<Expression> queryRangeSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(queryRangePredicate));
Set<Expression> viewRangeQueryBasedSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(viewRangePredicateQueryBased));
// remove unnecessary literal BooleanLiteral.TRUE
queryRangeSet.remove(BooleanLiteral.TRUE);
viewRangeQueryBasedSet.remove(BooleanLiteral.TRUE);
// query residual predicate can not contain all view residual predicate when view have residual predicate,
// bail out
if (!queryRangeSet.containsAll(viewRangeQueryBasedSet)) {
return null;
}
queryRangeSet.removeAll(viewRangeQueryBasedSet);
return queryRangeSet;
}
/**
* compensate residual predicates
*/
public static Set<Expression> compensateResidualPredicate(StructInfo queryStructInfo,
StructInfo viewStructInfo,
SlotMapping viewToQuerySlotMapping,
ComparisonResult comparisonResult) {
// TODO Residual predicates compensate, simplify implementation currently.
SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate();
SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate();
Expression queryResidualPredicate = querySplitPredicate.getResidualPredicate();
Expression viewResidualPredicate = viewSplitPredicate.getResidualPredicate();
Expression viewResidualPredicateQueryBased =
ExpressionUtils.replace(viewResidualPredicate, viewToQuerySlotMapping.toSlotReferenceMap());
Set<Expression> queryResidualSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(queryResidualPredicate));
Set<Expression> viewResidualQueryBasedSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(viewResidualPredicateQueryBased));
// remove unnecessary literal BooleanLiteral.TRUE
queryResidualSet.remove(BooleanLiteral.TRUE);
viewResidualQueryBasedSet.remove(BooleanLiteral.TRUE);
// query residual predicate can not contain all view residual predicate when view have residual predicate,
// bail out
if (!queryResidualSet.containsAll(viewResidualQueryBasedSet)) {
return null;
}
queryResidualSet.removeAll(viewResidualQueryBasedSet);
return queryResidualSet;
}
@Override
public String toString() {
return Utils.toSqlString("Predicates", "pulledUpPredicates", pulledUpPredicates);
@ -81,9 +210,11 @@ public class Predicates {
* The split different representation for predicate expression, such as equal, range and residual predicate.
*/
public static final class SplitPredicate {
private Optional<Expression> equalPredicate;
private Optional<Expression> rangePredicate;
private Optional<Expression> residualPredicate;
public static final SplitPredicate INVALID_INSTANCE =
SplitPredicate.of(null, null, null);
private final Optional<Expression> equalPredicate;
private final Optional<Expression> rangePredicate;
private final Optional<Expression> residualPredicate;
public SplitPredicate(Expression equalPredicate, Expression rangePredicate, Expression residualPredicate) {
this.equalPredicate = Optional.ofNullable(equalPredicate);
@ -103,10 +234,6 @@ public class Predicates {
return residualPredicate.orElse(BooleanLiteral.TRUE);
}
public static SplitPredicate invalid() {
return new SplitPredicate(null, null, null);
}
/**
* SplitPredicate construct
*/
@ -117,27 +244,23 @@ public class Predicates {
}
/**
* isEmpty
* Check the predicates are invalid or not. If any of the predicates is null, it is invalid.
*/
public boolean isEmpty() {
return !equalPredicate.isPresent()
&& !rangePredicate.isPresent()
&& !residualPredicate.isPresent();
public boolean isInvalid() {
return Objects.equals(this, INVALID_INSTANCE);
}
public List<Expression> toList() {
return ImmutableList.of(equalPredicate.orElse(BooleanLiteral.TRUE),
rangePredicate.orElse(BooleanLiteral.TRUE),
residualPredicate.orElse(BooleanLiteral.TRUE));
return ImmutableList.of(getEqualPredicate(), getRangePredicate(), getResidualPredicate());
}
/**
* Check the predicates in SplitPredicate is whether all true or not
*/
public boolean isAlwaysTrue() {
Expression equalExpr = equalPredicate.orElse(BooleanLiteral.TRUE);
Expression rangeExpr = rangePredicate.orElse(BooleanLiteral.TRUE);
Expression residualExpr = residualPredicate.orElse(BooleanLiteral.TRUE);
Expression equalExpr = getEqualPredicate();
Expression rangeExpr = getRangePredicate();
Expression residualExpr = getResidualPredicate();
return equalExpr instanceof BooleanLiteral
&& rangeExpr instanceof BooleanLiteral
&& residualExpr instanceof BooleanLiteral
@ -146,6 +269,25 @@ public class Predicates {
&& ((BooleanLiteral) residualExpr).getValue();
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
SplitPredicate that = (SplitPredicate) o;
return Objects.equals(equalPredicate, that.equalPredicate)
&& Objects.equals(rangePredicate, that.rangePredicate)
&& Objects.equals(residualPredicate, that.residualPredicate);
}
@Override
public int hashCode() {
return Objects.hash(equalPredicate, rangePredicate, residualPredicate);
}
@Override
public String toString() {
return Utils.toSqlString("SplitPredicate",

32
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
View File

@ -23,6 +23,7 @@ import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
@ -40,8 +41,10 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer;
import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@ -83,8 +86,11 @@ public class StructInfo {
// split predicates is shuttled
private SplitPredicate splitPredicate;
private EquivalenceClass equivalenceClass;
// this is for LogicalCompatibilityContext later
// Key is the expression shuttled and the value is the origin expression
// this is for building LogicalCompatibilityContext later.
private final Map<Expression, Expression> shuttledHashConjunctsToConjunctsMap = new HashMap<>();
// Record the exprId and the corresponding expr map, this is used by expression shuttled
private final Map<ExprId, Expression> namedExprIdAndExprMapping = new HashMap<>();
private StructInfo(Plan originalPlan, @Nullable Plan topPlan, @Nullable Plan bottomPlan, HyperGraph hyperGraph) {
this.originalPlan = originalPlan;
@ -117,12 +123,22 @@ public class StructInfo {
// Collect expression from join condition in hyper graph
this.hyperGraph.getJoinEdges().forEach(edge -> {
List<Expression> hashJoinConjuncts = edge.getHashJoinConjuncts();
// shuttle expression in edge for the build of LogicalCompatibilityContext later.
// Record the exprId to expr map in the processing to strut info
// TODO get exprId to expr map when complex project is ready in join dege
hashJoinConjuncts.forEach(conjunctExpr -> {
// shuttle expression in edge for LogicalCompatibilityContext later
shuttledHashConjunctsToConjunctsMap.put(
ExpressionUtils.shuttleExpressionWithLineage(
Lists.newArrayList(conjunctExpr), edge.getJoin()).get(0),
conjunctExpr);
ExpressionLineageReplacer.ExpressionReplaceContext replaceContext =
new ExpressionLineageReplacer.ExpressionReplaceContext(
Lists.newArrayList(conjunctExpr),
ImmutableSet.of(),
ImmutableSet.of());
this.topPlan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext);
// Replace expressions by expression map
List<Expression> replacedExpressions = replaceContext.getReplacedExpressions();
shuttledHashConjunctsToConjunctsMap.put(replacedExpressions.get(0), conjunctExpr);
// Record this, will be used in top level expression shuttle later, see the method
// ExpressionLineageReplacer#visitGroupPlan
this.namedExprIdAndExprMapping.putAll(replaceContext.getExprIdExpressionMap());
});
List<Expression> otherJoinConjuncts = edge.getOtherJoinConjuncts();
if (!otherJoinConjuncts.isEmpty()) {
@ -267,6 +283,10 @@ public class StructInfo {
return originalPlanId;
}
public Map<ExprId, Expression> getNamedExprIdAndExprMapping() {
return namedExprIdAndExprMapping;
}
/**
* Judge the source graph logical is whether the same as target
* For inner join should judge only the join tables,

25
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java
View File

@ -18,6 +18,8 @@
package org.apache.doris.nereids.trees.plans.visitor;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
@ -25,6 +27,7 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer.ExpressionReplaceContext;
@ -56,6 +59,22 @@ public class ExpressionLineageReplacer extends DefaultPlanVisitor<Expression, Ex
return super.visit(plan, context);
}
@Override
public Expression visitGroupPlan(GroupPlan groupPlan, ExpressionReplaceContext context) {
Group group = groupPlan.getGroup();
if (group == null) {
return visit(groupPlan, context);
}
List<StructInfo> structInfos = group.getStructInfos();
if (structInfos.isEmpty()) {
return visit(groupPlan, context);
}
// TODO only support group has one struct info, will support more struct info later
StructInfo structInfo = structInfos.get(0);
context.getExprIdExpressionMap().putAll(structInfo.getNamedExprIdAndExprMapping());
return visit(groupPlan, context);
}
/**
* Replace the expression with lineage according the exprIdExpressionMap
*/
@ -93,7 +112,7 @@ public class ExpressionLineageReplacer extends DefaultPlanVisitor<Expression, Ex
}
/**
* The Collector for target named expressions in the whole plan, and will be used to
* The Collector for named expressions in the whole plan, and will be used to
* replace the target expression later
* TODO Collect named expression by targetTypes, tableIdentifiers
*/
@ -128,7 +147,9 @@ public class ExpressionLineageReplacer extends DefaultPlanVisitor<Expression, Ex
private Map<ExprId, Expression> exprIdExpressionMap;
private List<Expression> replacedExpressions;
/**ExpressionReplaceContext*/
/**
* ExpressionReplaceContext
*/
public ExpressionReplaceContext(List<Expression> targetExpressions,
Set<TableType> targetTypes,
Set<String> tableIdentifiers) {