database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Fix](Nereids) Add cast comparison with slot reference when inferring predicate (#21171)

Browse Source 
Problem:
When inferring predicate, we assume that slot reference need to be inferred. But in this case:
carete table tb1(l1 smallint) ...;
create table tb2(l2 int) ...;
select * from tb1 inner join tb2 where tb1.l1 = tb2.l2 and tb2.l2 = 1;
We can not get tb1.l1 = 1 filter because we will add a cast to l1 (Cast smallint to int l1) = l2.

Solved:
Add cast consideration when inferring predicate, also add change judgement when judging equals to slotreference and cast expression. But when we want to infer predicate from bigger type cast to smaller type, it is logical error.
For example:
select * from tb1 inner join tb2 where tb1.l1 = cast(tb2.l2 as smallint) and tb2.l2 = (number between smallint max and intmax);
tb2.l2 value can not infer to left side because tb1.l1 would be false value, and when we add one more condition like tb1.l1 = tb3.l3(smallint). It would cause this predicate be false.
This commit is contained in:
LiBinfeng
2023-07-19 23:14:26 +08:00
committed by GitHub
parent c581cf311a
commit 58f2593ba1
4 changed files with 108 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PredicatePropagation.java
View File

@ -17,11 +17,15 @@
package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.coercion.IntegralType;
import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.Sets;
@ -70,19 +74,38 @@ public class PredicatePropagation {
@Override
public Expression visitComparisonPredicate(ComparisonPredicate cp, Void context) {
if (cp.left().isSlot() && cp.right().isConstant()) {
return replaceSlot(cp);
} else if (cp.left().isConstant() && cp.right().isSlot()) {
return replaceSlot(cp);
// we need to get expression covered by cast, because we want to infer different datatype
if (ExpressionUtils.isExpressionSlotCoveredByCast(cp.left()) && (cp.right().isConstant())) {
return replaceSlot(cp, ExpressionUtils.getDatatypeCoveredByCast(cp.left()));
} else if (ExpressionUtils.isExpressionSlotCoveredByCast(cp.right()) && cp.left().isConstant()) {
return replaceSlot(cp, ExpressionUtils.getDatatypeCoveredByCast(cp.right()));
}
return super.visit(cp, context);
}
private Expression replaceSlot(Expression expr) {
private boolean isOriginDataTypeBigger(DataType originDataType, Expression expr) {
if ((leftSlotEqualToRightSlot.child(0).getDataType() instanceof IntegralType)
&& (leftSlotEqualToRightSlot.child(1).getDataType() instanceof IntegralType)
&& (originDataType instanceof IntegralType)) {
// infer filter can not be lower than original datatype, or dataset would be wrong
if (((IntegralType) originDataType).widerThan(
(IntegralType) leftSlotEqualToRightSlot.child(0).getDataType())
|| ((IntegralType) originDataType).widerThan(
(IntegralType) leftSlotEqualToRightSlot.child(1).getDataType())) {
return true;
}
}
return false;
}
private Expression replaceSlot(Expression expr, DataType originDataType) {
return expr.rewriteUp(e -> {
if (e.equals(leftSlotEqualToRightSlot.child(0))) {
if (isOriginDataTypeBigger(originDataType, leftSlotEqualToRightSlot)) {
return e;
}
if (ExpressionUtils.isTwoExpressionEqualWithCast(e, leftSlotEqualToRightSlot.child(0))) {
return leftSlotEqualToRightSlot.child(1);
} else if (e.equals(leftSlotEqualToRightSlot.child(1))) {
} else if (ExpressionUtils.isTwoExpressionEqualWithCast(e, leftSlotEqualToRightSlot.child(1))) {
return leftSlotEqualToRightSlot.child(0);
} else {
return e;
@ -98,7 +121,8 @@ public class PredicatePropagation {
*/
private boolean canEquivalentInfer(Expression predicate) {
return predicate instanceof EqualTo
&& predicate.children().stream().allMatch(e -> e instanceof SlotReference)
&& predicate.children().stream().allMatch(e ->
(e instanceof SlotReference) || (e instanceof Cast && e.child(0).isSlot()))
&& predicate.child(0).getDataType().equals(predicate.child(1).getDataType());
}

4
fe/fe-core/src/main/java/org/apache/doris/nereids/types/coercion/IntegralType.java
View File

@ -41,4 +41,8 @@ public class IntegralType extends NumericType {
public String simpleString() {
return "integral";
}
public boolean widerThan(IntegralType other) {
return this.width() > other.width();
}
}

29
fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
View File

@ -38,6 +38,7 @@ import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
import org.apache.doris.nereids.types.DataType;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
@ -251,6 +252,34 @@ public class ExpressionUtils {
}
}
/**
* get slot covered by cast
* example: input: cast(cast(table.columnA)) output: columnA.datatype
*
*/
public static DataType getDatatypeCoveredByCast(Expression expr) {
if (expr instanceof Cast) {
return getDatatypeCoveredByCast(((Cast) expr).child());
}
return expr.getDataType();
}
/**
* judge if expression is slot covered by cast
* example: cast(cast(table.columnA))
*/
public static boolean isExpressionSlotCoveredByCast(Expression expr) {
if (expr instanceof Cast) {
return isExpressionSlotCoveredByCast(((Cast) expr).child());
}
return expr instanceof SlotReference;
}
public static boolean isTwoExpressionEqualWithCast(Expression left, Expression right) {
return ExpressionUtils.extractSlotOrCastOnSlot(left)
.equals(ExpressionUtils.extractSlotOrCastOnSlot(right));
}
/**
* Replace expression node in the expression tree by `replaceMap` in top-down manner.
* For example.