database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Fix](Nereids) Fix lost predicate when query has a filter at the right input of the outer join (#30374)

Browse Source 
materialized view def is as following:
>        select l_shipdate, o_orderdate, l_partkey, l_suppkey, o_orderkey  
>        from lineitem 
>        left join (select * from orders where o_orderdate = '2023-12-10' ) t2 
>        on lineitem.l_orderkey = t2.o_orderkey;
    
the query as following, should add filter `o_orderdate = '2023-12-10'` on mv when query rewrite by materialized view
>        select l_shipdate, o_orderdate, l_partkey, l_suppkey, o_orderkey 
>         from lineitem 
>         left join orders 
>        on lineitem.l_orderkey = orders.o_orderkey 
>         where o_orderdate = '2023-12-10' order by 1, 2, 3, 4, 5;
This commit is contained in:
seawinde
2024-01-26 17:41:48 +08:00
committed by yiguolei
parent 2befa75b9c
commit f25af15842
3 changed files with 73 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View File

@ -441,6 +441,25 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
viewStructInfo = viewStructInfo.withPredicates(
viewStructInfo.getPredicates().merge(viewPulledUpExpressions));
}
// if the join type in query and mv plan is different, we should check query is have the
// filters which rejects null
Set<Set<Slot>> requireNoNullableViewSlot = comparisonResult.getViewNoNullableSlot();
// check query is use the null reject slot which view comparison need
if (!requireNoNullableViewSlot.isEmpty()) {
SlotMapping queryToViewMapping = viewToQuerySlotMapping.inverse();
// try to use
boolean valid = containsNullRejectSlot(requireNoNullableViewSlot,
queryStructInfo.getPredicates().getPulledUpPredicates(), queryToViewMapping, cascadesContext);
if (!valid) {
queryStructInfo = queryStructInfo.withPredicates(
queryStructInfo.getPredicates().merge(comparisonResult.getQueryAllPulledUpExpressions()));
valid = containsNullRejectSlot(requireNoNullableViewSlot,
queryStructInfo.getPredicates().getPulledUpPredicates(), queryToViewMapping, cascadesContext);
}
if (!valid) {
return SplitPredicate.INVALID_INSTANCE;
}
}
// viewEquivalenceClass to query based
// equal predicate compensate
final Set<Expression> equalCompensateConjunctions = Predicates.compensateEquivalence(
@ -464,30 +483,6 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
|| residualCompensatePredicates == null) {
return SplitPredicate.INVALID_INSTANCE;
}
// if the join type in query and mv plan is different, we should check query is have the
// filters which rejects null
Set<Set<Slot>> requireNoNullableViewSlot = comparisonResult.getViewNoNullableSlot();
// check query is use the null reject slot which view comparison need
if (!requireNoNullableViewSlot.isEmpty()) {
Set<Expression> queryPulledUpPredicates = comparisonResult.getQueryAllPulledUpExpressions().stream()
.flatMap(expr -> ExpressionUtils.extractConjunction(expr).stream())
.collect(Collectors.toSet());
Set<Expression> nullRejectPredicates = ExpressionUtils.inferNotNull(queryPulledUpPredicates,
cascadesContext);
SlotMapping queryToViewMapping = viewToQuerySlotMapping.inverse();
Set<Expression> queryUsedNeedRejectNullSlotsViewBased = nullRejectPredicates.stream()
.map(expression -> TypeUtils.isNotNull(expression).orElse(null))
.filter(Objects::nonNull)
.map(expr -> ExpressionUtils.replace((Expression) expr, queryToViewMapping.toSlotReferenceMap()))
.collect(Collectors.toSet());
// query pulledUp predicates should have null reject predicates and contains any require noNullable slot
boolean valid = !queryPulledUpPredicates.containsAll(nullRejectPredicates)
&& requireNoNullableViewSlot.stream().noneMatch(
set -> Sets.intersection(set, queryUsedNeedRejectNullSlotsViewBased).isEmpty());
if (!valid) {
return SplitPredicate.INVALID_INSTANCE;
}
}
return SplitPredicate.of(equalCompensateConjunctions.isEmpty() ? BooleanLiteral.TRUE
: ExpressionUtils.and(equalCompensateConjunctions),
rangeCompensatePredicates.isEmpty() ? BooleanLiteral.TRUE
@ -496,6 +491,29 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
: ExpressionUtils.and(residualCompensatePredicates));
}
/**
* Check the queryPredicates contains the required nullable slot
*/
private boolean containsNullRejectSlot(Set<Set<Slot>> requireNoNullableViewSlot,
Set<Expression> queryPredicates,
SlotMapping queryToViewMapping,
CascadesContext cascadesContext) {
Set<Expression> queryPulledUpPredicates = queryPredicates.stream()
.flatMap(expr -> ExpressionUtils.extractConjunction(expr).stream())
.collect(Collectors.toSet());
Set<Expression> nullRejectPredicates = ExpressionUtils.inferNotNull(queryPulledUpPredicates,
cascadesContext);
Set<Expression> queryUsedNeedRejectNullSlotsViewBased = nullRejectPredicates.stream()
.map(expression -> TypeUtils.isNotNull(expression).orElse(null))
.filter(Objects::nonNull)
.map(expr -> ExpressionUtils.replace((Expression) expr, queryToViewMapping.toSlotReferenceMap()))
.collect(Collectors.toSet());
// query pulledUp predicates should have null reject predicates and contains any require noNullable slot
return !queryPulledUpPredicates.containsAll(nullRejectPredicates)
&& requireNoNullableViewSlot.stream().noneMatch(
set -> Sets.intersection(set, queryUsedNeedRejectNullSlotsViewBased).isEmpty());
}
/**
* Decide the match mode
*