[fix](Nereids) could not prune datev1 partition column (#24959)

because storage engine could not process date comparison predicates.
we convert it to datetime comparison predicates.
however, partition prunner could not process cast(slot) cp literal.
so, we convert back in partition pruner to let it work well.

TODO:
move convert date to datetime in translate stage
and only convert predicates for storage engine.
This commit is contained in:
morrySnow
2023-09-27 18:41:56 +08:00
committed by GitHub
parent a6bc0e7668
commit 00786a3295
4 changed files with 59 additions and 15 deletions

View File

@ -32,8 +32,8 @@ import java.util.Map;
* For any partition predicate, the evaluate() will always return true.
*/
public class HiveDefaultPartitionEvaluator implements OnePartitionEvaluator {
private long id;
private List<Slot> partitionSlots;
private final long id;
private final List<Slot> partitionSlots;
public HiveDefaultPartitionEvaluator(long id, List<Slot> partitionSlots) {
this.id = id;

View File

@ -76,15 +76,15 @@ public class OneRangePartitionEvaluator
extends ExpressionVisitor<EvaluateRangeResult, EvaluateRangeInput>
implements OnePartitionEvaluator {
private final long partitionId;
private List<Slot> partitionSlots;
private RangePartitionItem partitionItem;
private ExpressionRewriteContext expressionRewriteContext;
private List<PartitionSlotType> partitionSlotTypes;
private List<Literal> lowers;
private List<Literal> uppers;
private List<List<Expression>> inputs;
private Map<Slot, Boolean> partitionSlotContainsNull;
private Map<Slot, PartitionSlotType> slotToType;
private final List<Slot> partitionSlots;
private final RangePartitionItem partitionItem;
private final ExpressionRewriteContext expressionRewriteContext;
private final List<PartitionSlotType> partitionSlotTypes;
private final List<Literal> lowers;
private final List<Literal> uppers;
private final List<List<Expression>> inputs;
private final Map<Slot, Boolean> partitionSlotContainsNull;
private final Map<Slot, PartitionSlotType> slotToType;
/** OneRangePartitionEvaluator */
public OneRangePartitionEvaluator(long partitionId, List<Slot> partitionSlots,

View File

@ -22,9 +22,16 @@ import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.RangePartitionItem;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
import org.apache.doris.nereids.types.DateTimeType;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
@ -36,9 +43,9 @@ import java.util.Objects;
/**
* PartitionPruner
*/
public class PartitionPruner {
private List<OnePartitionEvaluator> partitions;
private Expression partitionPredicate;
public class PartitionPruner extends DefaultExpressionRewriter<Void> {
private final List<OnePartitionEvaluator> partitions;
private final Expression partitionPredicate;
/** Different type of table may have different partition prune behavior. */
public enum PartitionTableType {
@ -48,7 +55,40 @@ public class PartitionPruner {
private PartitionPruner(List<OnePartitionEvaluator> partitions, Expression partitionPredicate) {
this.partitions = Objects.requireNonNull(partitions, "partitions cannot be null");
this.partitionPredicate = Objects.requireNonNull(partitionPredicate, "partitionPredicate cannot be null");
this.partitionPredicate = Objects.requireNonNull(partitionPredicate.accept(this, null),
"partitionPredicate cannot be null");
}
@Override
public Expression visitComparisonPredicate(ComparisonPredicate cp, Void context) {
// Date cp Date is not supported in BE storage engine. So cast to DateTime in SimplifyComparisonPredicate
// for easy process partition prune, we convert back to date compare date here
// see more info in SimplifyComparisonPredicate
Expression left = cp.left();
Expression right = cp.right();
if (left.getDataType() != DateTimeType.INSTANCE || right.getDataType() != DateTimeType.INSTANCE) {
return cp;
}
if (!(left instanceof DateTimeLiteral) && !(right instanceof DateTimeLiteral)) {
return cp;
}
if (left instanceof DateTimeLiteral && ((DateTimeLiteral) left).isMidnight()
&& right instanceof Cast
&& ((Cast) right).child() instanceof SlotReference
&& ((Cast) right).child().getDataType().isDateType()) {
DateTimeLiteral dt = (DateTimeLiteral) left;
Cast cast = (Cast) right;
return cp.withChildren(new DateLiteral(dt.getYear(), dt.getMonth(), dt.getDay()), cast.child());
} else if (right instanceof DateTimeLiteral && ((DateTimeLiteral) right).isMidnight()
&& left instanceof Cast
&& ((Cast) left).child() instanceof SlotReference
&& ((Cast) left).child().getDataType().isDateType()) {
DateTimeLiteral dt = (DateTimeLiteral) right;
Cast cast = (Cast) left;
return cp.withChildren(cast.child(), new DateLiteral(dt.getYear(), dt.getMonth(), dt.getDay()));
} else {
return cp;
}
}
public List<Long> prune() {

View File

@ -93,6 +93,10 @@ public class DateTimeLiteral extends DateLiteral {
this.day = day;
}
public boolean isMidnight() {
return hour == 0 && minute == 0 && second == 0 && microSecond == 0;
}
/**
* determine scale by datetime string
*/