database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](nereids) Composite predicate supports range predicate when rewritting by materialzied view (#31538)

Browse Source 
It supports predicate composite as following:
materialized view define
>        select l_shipdate, o_orderdate, l_partkey, l_suppkey
>        from lineitem_1
>        left join orders_1
>        on lineitem_1.l_orderkey = orders_1.o_orderkey
>        where l_shipdate > '2023-10-19'


the query as following can be rewritten by the materialized view above
>        select l_shipdate, o_orderdate, l_partkey, l_suppkey
>        from lineitem_1
>        left join orders_1
>        on lineitem_1.l_orderkey = orders_1.o_orderkey
>        where l_shipdate > '2023-10-25'
This commit is contained in:
seawinde
2024-03-01 18:03:10 +08:00
committed by yiguolei
parent 493c9d49ea
commit b26dcf2677
8 changed files with 255 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
View File

@ -473,7 +473,8 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
queryStructInfo,
viewStructInfo,
viewToQuerySlotMapping,
comparisonResult);
comparisonResult,
cascadesContext);
// residual compensate
final Set<Expression> residualCompensatePredicates = Predicates.compensateResidualPredicate(
queryStructInfo,

3
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
View File

@ -75,6 +75,9 @@ public class MaterializedViewUtils {
break;
}
}
if (columnExpr == null) {
return Optional.empty();
}
if (!(columnExpr instanceof SlotReference)) {
return Optional.empty();
}

44
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java
View File

@ -17,8 +17,12 @@
package org.apache.doris.nereids.rules.exploration.mv;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.exploration.mv.mapping.EquivalenceClassSetMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
import org.apache.doris.nereids.rules.expression.ExpressionNormalization;
import org.apache.doris.nereids.rules.expression.ExpressionOptimization;
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
@ -143,8 +147,8 @@ public class Predicates {
public static Set<Expression> compensateRangePredicate(StructInfo queryStructInfo,
StructInfo viewStructInfo,
SlotMapping viewToQuerySlotMapping,
ComparisonResult comparisonResult) {
// TODO Range predicates compensate, simplify implementation currently.
ComparisonResult comparisonResult,
CascadesContext cascadesContext) {
SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate();
SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate();
@ -153,20 +157,32 @@ public class Predicates {
Expression viewRangePredicateQueryBased =
ExpressionUtils.replace(viewRangePredicate, viewToQuerySlotMapping.toSlotReferenceMap());
Set<Expression> queryRangeSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(queryRangePredicate));
Set<Expression> viewRangeQueryBasedSet =
Sets.newHashSet(ExpressionUtils.extractConjunction(viewRangePredicateQueryBased));
// remove unnecessary literal BooleanLiteral.TRUE
queryRangeSet.remove(BooleanLiteral.TRUE);
viewRangeQueryBasedSet.remove(BooleanLiteral.TRUE);
// query residual predicate can not contain all view residual predicate when view have residual predicate,
// bail out
if (!queryRangeSet.containsAll(viewRangeQueryBasedSet)) {
Set<Expression> queryRangeSet = ExpressionUtils.extractConjunctionToSet(queryRangePredicate);
Set<Expression> viewRangeQueryBasedSet = ExpressionUtils.extractConjunctionToSet(viewRangePredicateQueryBased);
Set<Expression> differentExpressions = new HashSet<>();
Sets.difference(queryRangeSet, viewRangeQueryBasedSet).copyInto(differentExpressions);
Sets.difference(viewRangeQueryBasedSet, queryRangeSet).copyInto(differentExpressions);
// the range predicate in query and view is same, don't need to compensate
if (differentExpressions.isEmpty()) {
return differentExpressions;
}
// try to normalize the different expressions
Set<Expression> normalizedExpressions =
normalizeExpression(ExpressionUtils.and(differentExpressions), cascadesContext);
if (!queryRangeSet.containsAll(normalizedExpressions)) {
// normalized expressions is not in query, can not compensate
return null;
}
queryRangeSet.removeAll(viewRangeQueryBasedSet);
return queryRangeSet;
return normalizedExpressions;
}
private static Set<Expression> normalizeExpression(Expression expression, CascadesContext cascadesContext) {
ExpressionNormalization expressionNormalization = new ExpressionNormalization();
ExpressionOptimization expressionOptimization = new ExpressionOptimization();
ExpressionRewriteContext context = new ExpressionRewriteContext(cascadesContext);
expression = expressionNormalization.rewrite(expression, context);
expression = expressionOptimization.rewrite(expression, context);
return ExpressionUtils.extractConjunctionToSet(expression);
}
/**

12
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java
View File

@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
import org.apache.doris.nereids.trees.expressions.EqualPredicate;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.InPredicate;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
@ -81,6 +82,17 @@ public class PredicatesSplitter {
return null;
}
@Override
public Void visitInPredicate(InPredicate inPredicate, Void context) {
if (containOnlyColumnRef(inPredicate.getCompareExpr(), true)
&& (ExpressionUtils.isAllLiteral(inPredicate.getOptions()))) {
rangePredicates.add(inPredicate);
} else {
residualPredicates.add(inPredicate);
}
return null;
}
@Override
public Void visit(Expression expr, Void context) {
residualPredicates.add(expr);

9
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java
View File

@ -98,8 +98,8 @@ public class SimplifyRange extends AbstractExpressionRewriteRule {
private ValueDesc buildRange(ComparisonPredicate predicate) {
Expression rewrite = ExpressionRuleExecutor.normalize(predicate);
Expression right = rewrite.child(1);
// only handle `NumericType`
if (right.isLiteral() && right.getDataType().isNumericType()) {
// only handle `NumericType` and `DateLikeType`
if (right.isLiteral() && (right.getDataType().isNumericType() || right.getDataType().isDateLikeType())) {
return ValueDesc.range((ComparisonPredicate) rewrite);
}
return new UnknownValue(predicate);
@ -132,9 +132,10 @@ public class SimplifyRange extends AbstractExpressionRewriteRule {
@Override
public ValueDesc visitInPredicate(InPredicate inPredicate, Void context) {
// only handle `NumericType`
// only handle `NumericType` and `DateLikeType`
if (ExpressionUtils.isAllLiteral(inPredicate.getOptions())
&& ExpressionUtils.matchNumericType(inPredicate.getOptions())) {
&& (ExpressionUtils.matchNumericType(inPredicate.getOptions())
|| ExpressionUtils.matchDateLikeType(inPredicate.getOptions()))) {
return ValueDesc.discrete(inPredicate);
}
return new UnknownValue(inPredicate);

4
fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
View File

@ -467,6 +467,10 @@ public class ExpressionUtils {
return children.stream().allMatch(c -> c.getDataType().isNumericType());
}
public static boolean matchDateLikeType(List<Expression> children) {
return children.stream().allMatch(c -> c.getDataType().isDateLikeType());
}
public static boolean hasNullLiteral(List<Expression> children) {
return children.stream().anyMatch(c -> c instanceof NullLiteral);
}