From 00786a3295c53a75af188f704e954d2a37e028b3 Mon Sep 17 00:00:00 2001 From: morrySnow <101034200+morrySnow@users.noreply.github.com> Date: Wed, 27 Sep 2023 18:41:56 +0800 Subject: [PATCH] [fix](Nereids) could not prune datev1 partition column (#24959) because storage engine could not process date comparison predicates. we convert it to datetime comparison predicates. however, partition prunner could not process cast(slot) cp literal. so, we convert back in partition pruner to let it work well. TODO: move convert date to datetime in translate stage and only convert predicates for storage engine. --- .../rules/HiveDefaultPartitionEvaluator.java | 4 +- .../rules/OneRangePartitionEvaluator.java | 18 +++---- .../expression/rules/PartitionPruner.java | 48 +++++++++++++++++-- .../expressions/literal/DateTimeLiteral.java | 4 ++ 4 files changed, 59 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/HiveDefaultPartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/HiveDefaultPartitionEvaluator.java index 04785e1db1..f50cbe3cf3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/HiveDefaultPartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/HiveDefaultPartitionEvaluator.java @@ -32,8 +32,8 @@ import java.util.Map; * For any partition predicate, the evaluate() will always return true. */ public class HiveDefaultPartitionEvaluator implements OnePartitionEvaluator { - private long id; - private List partitionSlots; + private final long id; + private final List partitionSlots; public HiveDefaultPartitionEvaluator(long id, List partitionSlots) { this.id = id; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java index f985764685..13ecddd150 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java @@ -76,15 +76,15 @@ public class OneRangePartitionEvaluator extends ExpressionVisitor implements OnePartitionEvaluator { private final long partitionId; - private List partitionSlots; - private RangePartitionItem partitionItem; - private ExpressionRewriteContext expressionRewriteContext; - private List partitionSlotTypes; - private List lowers; - private List uppers; - private List> inputs; - private Map partitionSlotContainsNull; - private Map slotToType; + private final List partitionSlots; + private final RangePartitionItem partitionItem; + private final ExpressionRewriteContext expressionRewriteContext; + private final List partitionSlotTypes; + private final List lowers; + private final List uppers; + private final List> inputs; + private final Map partitionSlotContainsNull; + private final Map slotToType; /** OneRangePartitionEvaluator */ public OneRangePartitionEvaluator(long partitionId, List partitionSlots, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java index 8479c27776..6c932f558b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java @@ -22,9 +22,16 @@ import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.RangePartitionItem; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; +import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; +import org.apache.doris.nereids.types.DateTimeType; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -36,9 +43,9 @@ import java.util.Objects; /** * PartitionPruner */ -public class PartitionPruner { - private List partitions; - private Expression partitionPredicate; +public class PartitionPruner extends DefaultExpressionRewriter { + private final List partitions; + private final Expression partitionPredicate; /** Different type of table may have different partition prune behavior. */ public enum PartitionTableType { @@ -48,7 +55,40 @@ public class PartitionPruner { private PartitionPruner(List partitions, Expression partitionPredicate) { this.partitions = Objects.requireNonNull(partitions, "partitions cannot be null"); - this.partitionPredicate = Objects.requireNonNull(partitionPredicate, "partitionPredicate cannot be null"); + this.partitionPredicate = Objects.requireNonNull(partitionPredicate.accept(this, null), + "partitionPredicate cannot be null"); + } + + @Override + public Expression visitComparisonPredicate(ComparisonPredicate cp, Void context) { + // Date cp Date is not supported in BE storage engine. So cast to DateTime in SimplifyComparisonPredicate + // for easy process partition prune, we convert back to date compare date here + // see more info in SimplifyComparisonPredicate + Expression left = cp.left(); + Expression right = cp.right(); + if (left.getDataType() != DateTimeType.INSTANCE || right.getDataType() != DateTimeType.INSTANCE) { + return cp; + } + if (!(left instanceof DateTimeLiteral) && !(right instanceof DateTimeLiteral)) { + return cp; + } + if (left instanceof DateTimeLiteral && ((DateTimeLiteral) left).isMidnight() + && right instanceof Cast + && ((Cast) right).child() instanceof SlotReference + && ((Cast) right).child().getDataType().isDateType()) { + DateTimeLiteral dt = (DateTimeLiteral) left; + Cast cast = (Cast) right; + return cp.withChildren(new DateLiteral(dt.getYear(), dt.getMonth(), dt.getDay()), cast.child()); + } else if (right instanceof DateTimeLiteral && ((DateTimeLiteral) right).isMidnight() + && left instanceof Cast + && ((Cast) left).child() instanceof SlotReference + && ((Cast) left).child().getDataType().isDateType()) { + DateTimeLiteral dt = (DateTimeLiteral) right; + Cast cast = (Cast) left; + return cp.withChildren(cast.child(), new DateLiteral(dt.getYear(), dt.getMonth(), dt.getDay())); + } else { + return cp; + } } public List prune() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java index 3318951c95..aabe6f1e19 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java @@ -93,6 +93,10 @@ public class DateTimeLiteral extends DateLiteral { this.day = day; } + public boolean isMidnight() { + return hour == 0 && minute == 0 && second == 0 && microSecond == 0; + } + /** * determine scale by datetime string */