database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Feat](nereids) Support date_trunc function in partition prune (#38025) (#38849)

Browse Source 
cherry-pick #38025 to branch-2.1
This commit is contained in:
feiniaofeiafei
2024-08-05 18:29:10 +08:00
committed by GitHub
parent 9d5af7febd
commit 0f0b0e9b37
6 changed files with 560 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/ColumnRange.java
View File

@ -138,4 +138,12 @@ public class ColumnRange {
public static ColumnRange range(Literal lower, BoundType lowerType, Literal upper, BoundType upperType) {
return new ColumnRange(ColumnBound.range(lower, lowerType, upper, upperType));
}
public ColumnRange withLowerBound(Literal lower) {
return this.intersect(atLeast(lower));
}
public ColumnRange withUpperBound(Literal upper) {
return this.intersect(atMost(upper));
}
}

315
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
View File

@ -42,16 +42,18 @@ import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.functions.Monotonic;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Date;
import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.BooleanType;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Preconditions;
import com.google.common.collect.BoundType;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@ -62,6 +64,7 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Range;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@ -87,8 +90,9 @@ public class OneRangePartitionEvaluator
private final List<Literal> lowers;
private final List<Literal> uppers;
private final List<List<Expression>> inputs;
private final Map<Slot, Boolean> partitionSlotContainsNull;
private final Map<Expression, Boolean> partitionSlotContainsNull;
private final Map<Slot, PartitionSlotType> slotToType;
private final Map<Expression, ColumnRange> rangeMap = new HashMap<>();
/** OneRangePartitionEvaluator */
public OneRangePartitionEvaluator(long partitionId, List<Slot> partitionSlots,
@ -109,8 +113,8 @@ public class OneRangePartitionEvaluator
// fast path
Slot partSlot = partitionSlots.get(0);
this.slotToType = ImmutableMap.of(partSlot, partitionSlotTypes.get(0));
this.partitionSlotContainsNull
= ImmutableMap.of(partSlot, range.lowerEndpoint().getKeys().get(0).isMinValue());
this.partitionSlotContainsNull = new HashMap<>();
partitionSlotContainsNull.put(partSlot, range.lowerEndpoint().getKeys().get(0).isMinValue());
} else {
// slow path
this.slotToType = Maps.newHashMap();
@ -169,9 +173,9 @@ public class OneRangePartitionEvaluator
@Override
public Expression evaluate(Expression expression, Map<Slot, PartitionSlotInput> currentInputs) {
Map<Slot, ColumnRange> defaultColumnRanges = currentInputs.values().iterator().next().columnRanges;
EvaluateRangeResult result = expression.accept(
this, new EvaluateRangeInput(defaultColumnRanges, currentInputs));
Map<Expression, ColumnRange> defaultColumnRanges = currentInputs.values().iterator().next().columnRanges;
rangeMap.putAll(defaultColumnRanges);
EvaluateRangeResult result = expression.accept(this, new EvaluateRangeInput(currentInputs));
return result.result;
}
@ -194,22 +198,14 @@ public class OneRangePartitionEvaluator
return result;
}
@Override
public EvaluateRangeResult visitNullLiteral(NullLiteral nullLiteral, EvaluateRangeInput context) {
Map<Slot, ColumnRange> emptyRanges = Maps.newHashMap();
for (Slot key : context.defaultColumnRanges.keySet()) {
emptyRanges.put(key, new ColumnRange());
}
return new EvaluateRangeResult(nullLiteral, emptyRanges, ImmutableList.of());
}
@Override
public EvaluateRangeResult visitSlot(Slot slot, EvaluateRangeInput context) {
// try to replace partition slot to literal
PartitionSlotInput slotResult = context.slotToInput.get(slot);
return slotResult == null
? new EvaluateRangeResult(slot, context.defaultColumnRanges, ImmutableList.of())
: new EvaluateRangeResult(slotResult.result, slotResult.columnRanges, ImmutableList.of());
Preconditions.checkState(slotResult != null);
Preconditions.checkState(slotResult.columnRanges.containsKey(slot));
return new EvaluateRangeResult(slotResult.result, ImmutableMap.of(slot, slotResult.columnRanges.get(slot)),
ImmutableList.of());
}
@Override
@ -219,19 +215,19 @@ public class OneRangePartitionEvaluator
return result;
}
greaterThan = (GreaterThan) result.result;
if (greaterThan.left() instanceof Slot && greaterThan.right() instanceof Literal) {
Slot slot = (Slot) greaterThan.left();
if (isPartitionSlot(slot)) {
Map<Slot, ColumnRange> leftColumnRanges = result.childrenResult.get(0).columnRanges;
if (!(greaterThan.left() instanceof Literal) && greaterThan.right() instanceof Literal) {
Expression expr = greaterThan.left();
Map<Expression, ColumnRange> leftColumnRanges = result.childrenResult.get(0).columnRanges;
if (leftColumnRanges.containsKey(expr)) {
ColumnRange greaterThenRange = ColumnRange.greaterThan((Literal) greaterThan.right());
result = intersectSlotRange(result, leftColumnRanges, slot, greaterThenRange);
result = intersectSlotRange(result, leftColumnRanges, expr, greaterThenRange);
}
} else if (greaterThan.left() instanceof Literal && greaterThan.right() instanceof Slot) {
Slot slot = (Slot) greaterThan.right();
if (isPartitionSlot(slot)) {
Map<Slot, ColumnRange> rightColumnRanges = result.childrenResult.get(1).columnRanges;
} else if (greaterThan.left() instanceof Literal && !(greaterThan.right() instanceof Literal)) {
Expression expr = greaterThan.right();
Map<Expression, ColumnRange> rightColumnRanges = result.childrenResult.get(1).columnRanges;
if (rightColumnRanges.containsKey(expr)) {
ColumnRange lessThenRange = ColumnRange.lessThen((Literal) greaterThan.left());
result = intersectSlotRange(result, rightColumnRanges, slot, lessThenRange);
result = intersectSlotRange(result, rightColumnRanges, expr, lessThenRange);
}
}
return result;
@ -244,19 +240,19 @@ public class OneRangePartitionEvaluator
return result;
}
greaterThanEqual = (GreaterThanEqual) result.result;
if (greaterThanEqual.left() instanceof Slot && greaterThanEqual.right() instanceof Literal) {
Slot slot = (Slot) greaterThanEqual.left();
if (isPartitionSlot(slot)) {
Map<Slot, ColumnRange> leftColumnRanges = result.childrenResult.get(0).columnRanges;
if (!(greaterThanEqual.left() instanceof Literal) && greaterThanEqual.right() instanceof Literal) {
Expression expr = greaterThanEqual.left();
Map<Expression, ColumnRange> leftColumnRanges = result.childrenResult.get(0).columnRanges;
if (leftColumnRanges.containsKey(expr)) {
ColumnRange atLeastRange = ColumnRange.atLeast((Literal) greaterThanEqual.right());
result = intersectSlotRange(result, leftColumnRanges, slot, atLeastRange);
result = intersectSlotRange(result, leftColumnRanges, expr, atLeastRange);
}
} else if (greaterThanEqual.left() instanceof Literal && greaterThanEqual.right() instanceof Slot) {
Slot slot = (Slot) greaterThanEqual.right();
if (isPartitionSlot(slot)) {
Map<Slot, ColumnRange> rightColumnRanges = result.childrenResult.get(1).columnRanges;
} else if (greaterThanEqual.left() instanceof Literal && !(greaterThanEqual.right() instanceof Literal)) {
Expression expr = greaterThanEqual.right();
Map<Expression, ColumnRange> rightColumnRanges = result.childrenResult.get(1).columnRanges;
if (rightColumnRanges.containsKey(expr)) {
ColumnRange atMostRange = ColumnRange.atMost((Literal) greaterThanEqual.left());
result = intersectSlotRange(result, rightColumnRanges, slot, atMostRange);
result = intersectSlotRange(result, rightColumnRanges, expr, atMostRange);
}
}
return result;
@ -269,19 +265,19 @@ public class OneRangePartitionEvaluator
return result;
}
lessThan = (LessThan) result.result;
if (lessThan.left() instanceof Slot && lessThan.right() instanceof Literal) {
Slot slot = (Slot) lessThan.left();
if (isPartitionSlot(slot)) {
Map<Slot, ColumnRange> leftColumnRanges = result.childrenResult.get(0).columnRanges;
if (!(lessThan.left() instanceof Literal) && lessThan.right() instanceof Literal) {
Expression expr = lessThan.left();
Map<Expression, ColumnRange> leftColumnRanges = result.childrenResult.get(0).columnRanges;
if (leftColumnRanges.containsKey(expr)) {
ColumnRange greaterThenRange = ColumnRange.lessThen((Literal) lessThan.right());
result = intersectSlotRange(result, leftColumnRanges, slot, greaterThenRange);
result = intersectSlotRange(result, leftColumnRanges, expr, greaterThenRange);
}
} else if (lessThan.left() instanceof Literal && lessThan.right() instanceof Slot) {
Slot slot = (Slot) lessThan.right();
if (isPartitionSlot(slot)) {
Map<Slot, ColumnRange> rightColumnRanges = result.childrenResult.get(1).columnRanges;
} else if (lessThan.left() instanceof Literal && !(lessThan.right() instanceof Literal)) {
Expression expr = lessThan.right();
Map<Expression, ColumnRange> rightColumnRanges = result.childrenResult.get(1).columnRanges;
if (rightColumnRanges.containsKey(expr)) {
ColumnRange lessThenRange = ColumnRange.greaterThan((Literal) lessThan.left());
result = intersectSlotRange(result, rightColumnRanges, slot, lessThenRange);
result = intersectSlotRange(result, rightColumnRanges, expr, lessThenRange);
}
}
return result;
@ -294,19 +290,19 @@ public class OneRangePartitionEvaluator
return result;
}
lessThanEqual = (LessThanEqual) result.result;
if (lessThanEqual.left() instanceof Slot && lessThanEqual.right() instanceof Literal) {
Slot slot = (Slot) lessThanEqual.left();
if (isPartitionSlot(slot)) {
Map<Slot, ColumnRange> leftColumnRanges = result.childrenResult.get(0).columnRanges;
if (!(lessThanEqual.left() instanceof Literal) && lessThanEqual.right() instanceof Literal) {
Expression expr = lessThanEqual.left();
Map<Expression, ColumnRange> leftColumnRanges = result.childrenResult.get(0).columnRanges;
if (leftColumnRanges.containsKey(expr)) {
ColumnRange atLeastRange = ColumnRange.atMost((Literal) lessThanEqual.right());
result = intersectSlotRange(result, leftColumnRanges, slot, atLeastRange);
result = intersectSlotRange(result, leftColumnRanges, expr, atLeastRange);
}
} else if (lessThanEqual.left() instanceof Literal && lessThanEqual.right() instanceof Slot) {
Slot slot = (Slot) lessThanEqual.right();
if (isPartitionSlot(slot)) {
Map<Slot, ColumnRange> rightColumnRanges = result.childrenResult.get(1).columnRanges;
} else if (lessThanEqual.left() instanceof Literal && !(lessThanEqual.right() instanceof Literal)) {
Expression expr = lessThanEqual.right();
Map<Expression, ColumnRange> rightColumnRanges = result.childrenResult.get(1).columnRanges;
if (rightColumnRanges.containsKey(expr)) {
ColumnRange atMostRange = ColumnRange.atLeast((Literal) lessThanEqual.left());
result = intersectSlotRange(result, rightColumnRanges, slot, atMostRange);
result = intersectSlotRange(result, rightColumnRanges, expr, atMostRange);
}
}
return result;
@ -319,23 +315,23 @@ public class OneRangePartitionEvaluator
return result;
}
boolean isRejectNot = false;
if (equalTo.left() instanceof Slot && equalTo.right() instanceof Literal) {
Slot slot = (Slot) equalTo.left();
if (isPartitionSlot(slot)) {
Map<Slot, ColumnRange> leftColumnRanges = result.childrenResult.get(0).columnRanges;
if (!(equalTo.left() instanceof Literal) && equalTo.right() instanceof Literal) {
Expression expr = equalTo.left();
Map<Expression, ColumnRange> leftColumnRanges = result.childrenResult.get(0).columnRanges;
if (leftColumnRanges.containsKey(expr)) {
ColumnRange atLeastRange = ColumnRange.singleton((Literal) equalTo.right());
result = intersectSlotRange(result, leftColumnRanges, slot, atLeastRange);
if (leftColumnRanges.get(slot).isSingleton()) {
result = intersectSlotRange(result, leftColumnRanges, expr, atLeastRange);
if (leftColumnRanges.get(expr).isSingleton()) {
isRejectNot = true;
}
}
} else if (equalTo.left() instanceof Literal && equalTo.right() instanceof Slot) {
Slot slot = (Slot) equalTo.right();
if (isPartitionSlot(slot)) {
Map<Slot, ColumnRange> rightColumnRanges = result.childrenResult.get(1).columnRanges;
} else if (equalTo.left() instanceof Literal && !(equalTo.right() instanceof Literal)) {
Expression expr = equalTo.right();
Map<Expression, ColumnRange> rightColumnRanges = result.childrenResult.get(1).columnRanges;
if (rightColumnRanges.containsKey(expr)) {
ColumnRange atMostRange = ColumnRange.singleton((Literal) equalTo.left());
result = intersectSlotRange(result, rightColumnRanges, slot, atMostRange);
if (rightColumnRanges.get(slot).isSingleton()) {
result = intersectSlotRange(result, rightColumnRanges, expr, atMostRange);
if (rightColumnRanges.get(expr).isSingleton()) {
isRejectNot = true;
}
}
@ -356,11 +352,11 @@ public class OneRangePartitionEvaluator
}
// "A <=> null" has been convert to "A is null" or false by NullSafeEqualToEqual rule
// so we don't consider "A <=> null" here
if (nullSafeEqual.left() instanceof Slot && nullSafeEqual.right() instanceof Literal) {
if (!(nullSafeEqual.left() instanceof Literal) && nullSafeEqual.right() instanceof Literal) {
// A <=> literal -> A = literal and A is not null
return visit(ExpressionUtils.and(new EqualTo(nullSafeEqual.left(), nullSafeEqual.right()),
new Not(new IsNull(nullSafeEqual.left()))), context);
} else if (nullSafeEqual.left() instanceof Literal && nullSafeEqual.right() instanceof Slot) {
} else if (nullSafeEqual.left() instanceof Literal && !(nullSafeEqual.right() instanceof Slot)) {
// literal <=> A -> literal = A and A is not null
return visit(ExpressionUtils.and(new EqualTo(nullSafeEqual.left(), nullSafeEqual.right()),
new Not(new IsNull(nullSafeEqual.right()))), context);
@ -376,17 +372,17 @@ public class OneRangePartitionEvaluator
return result;
}
inPredicate = (InPredicate) result.result;
if (inPredicate.getCompareExpr() instanceof Slot
Map<Expression, ColumnRange> exprRanges = result.childrenResult.get(0).columnRanges;
if (exprRanges.containsKey(inPredicate.getCompareExpr())
&& inPredicate.getOptions().stream().allMatch(Literal.class::isInstance)) {
Slot slot = (Slot) inPredicate.getCompareExpr();
Expression compareExpr = inPredicate.getCompareExpr();
ColumnRange unionLiteralRange = ColumnRange.empty();
ColumnRange slotRange = result.childrenResult.get(0).columnRanges.get(slot);
ColumnRange compareExprRange = result.childrenResult.get(0).columnRanges.get(compareExpr);
for (Expression expr : inPredicate.getOptions()) {
unionLiteralRange = unionLiteralRange.union(
slotRange.intersect(ColumnRange.singleton((Literal) expr)));
compareExprRange.intersect(ColumnRange.singleton((Literal) expr)));
}
Map<Slot, ColumnRange> slotRanges = result.childrenResult.get(0).columnRanges;
result = intersectSlotRange(result, slotRanges, slot, unionLiteralRange);
result = intersectSlotRange(result, exprRanges, compareExpr, unionLiteralRange);
}
result = result.withRejectNot(false);
return result;
@ -400,11 +396,10 @@ public class OneRangePartitionEvaluator
}
result = result.withRejectNot(false);
Expression child = isNull.child();
if (!(child instanceof Slot) || !isPartitionSlot((Slot) child)) {
if (!partitionSlotContainsNull.containsKey(child)) {
return result;
}
if (!partitionSlotContainsNull.get((Slot) child)) {
if (!partitionSlotContainsNull.get(child)) {
return new EvaluateRangeResult(BooleanLiteral.FALSE,
result.columnRanges, result.childrenResult, false);
}
@ -416,7 +411,15 @@ public class OneRangePartitionEvaluator
EvaluateRangeResult result = evaluateChildrenThenThis(and, context);
result = mergeRanges(result.result, result.childrenResult.get(0), result.childrenResult.get(1),
(leftRange, rightRange) -> leftRange.intersect(rightRange));
(leftRange, rightRange) -> {
if (leftRange == null) {
return rightRange;
}
if (rightRange == null) {
return leftRange;
}
return leftRange.intersect(rightRange);
});
result = returnFalseIfExistEmptyRange(result);
if (result.result.equals(BooleanLiteral.FALSE)) {
@ -434,20 +437,29 @@ public class OneRangePartitionEvaluator
EvaluateRangeResult result = evaluateChildrenThenThis(or, context);
result = mergeRanges(result.result, result.childrenResult.get(0), result.childrenResult.get(1),
(leftRange, rightRange) -> leftRange.union(rightRange));
return returnFalseIfExistEmptyRange(result);
(leftRange, rightRange) -> {
if (leftRange == null) {
return rightRange;
}
if (rightRange == null) {
return leftRange;
}
return leftRange.union(rightRange);
});
return removeEmptyRange(result);
}
@Override
public EvaluateRangeResult visitNot(Not not, EvaluateRangeInput context) {
EvaluateRangeResult result = evaluateChildrenThenThis(not, context);
if (result.isRejectNot() && !result.result.equals(BooleanLiteral.TRUE)) {
Map<Slot, ColumnRange> newRanges = Maps.newHashMap();
for (Map.Entry<Slot, ColumnRange> entry : result.childrenResult.get(0).columnRanges.entrySet()) {
Slot slot = entry.getKey();
Map<Expression, ColumnRange> newRanges = Maps.newHashMap();
for (Map.Entry<Expression, ColumnRange> entry : result.childrenResult.get(0).columnRanges.entrySet()) {
Expression expr = entry.getKey();
ColumnRange childRange = entry.getValue();
ColumnRange partitionRange = result.columnRanges.get(slot);
newRanges.put(slot, partitionRange.intersect(childRange.complete()));
ColumnRange partitionRange = rangeMap.containsKey(expr)
? rangeMap.get(expr) : ColumnRange.all();
newRanges.put(expr, partitionRange.intersect(childRange.complete()));
}
result = new EvaluateRangeResult(result.result, newRanges, result.childrenResult);
}
@ -476,7 +488,7 @@ public class OneRangePartitionEvaluator
// evaluate this
expr = FoldConstantRuleOnFE.evaluate(expr, expressionRewriteContext);
return new EvaluateRangeResult(expr, context.defaultColumnRanges, childrenResults);
return new EvaluateRangeResult(expr, ImmutableMap.of(), childrenResults);
}
private EvaluateRangeResult returnFalseIfExistEmptyRange(EvaluateRangeResult result) {
@ -489,11 +501,11 @@ public class OneRangePartitionEvaluator
}
private EvaluateRangeResult intersectSlotRange(EvaluateRangeResult originResult,
Map<Slot, ColumnRange> columnRanges, Slot slot, ColumnRange otherRange) {
ColumnRange columnRange = columnRanges.get(slot);
Map<Expression, ColumnRange> columnRanges, Expression expr, ColumnRange otherRange) {
ColumnRange columnRange = columnRanges.get(expr);
ColumnRange intersect = columnRange.intersect(otherRange);
Map<Slot, ColumnRange> newColumnRanges = replaceSlotRange(columnRanges, slot, intersect);
Map<Expression, ColumnRange> newColumnRanges = replaceExprRange(columnRanges, expr, intersect);
if (intersect.isEmptyRange()) {
return new EvaluateRangeResult(BooleanLiteral.FALSE, newColumnRanges, originResult.childrenResult);
@ -513,6 +525,9 @@ public class OneRangePartitionEvaluator
for (int i = 0; i < partitionSlotTypes.size(); i++) {
PartitionSlotType partitionSlotType = partitionSlotTypes.get(i);
Slot slot = partitionSlots.get(i);
if (!context.columnRanges.containsKey(slot)) {
return context;
}
switch (partitionSlotType) {
case CONST: continue;
case RANGE:
@ -547,7 +562,7 @@ public class OneRangePartitionEvaluator
ColumnRange origin = context.columnRanges.get(qualifiedSlot);
ColumnRange newRange = origin.intersect(qualifiedRange);
Map<Slot, ColumnRange> newRanges = replaceSlotRange(context.columnRanges, qualifiedSlot, newRange);
Map<Expression, ColumnRange> newRanges = replaceExprRange(context.columnRanges, qualifiedSlot, newRange);
if (newRange.isEmptyRange()) {
return new EvaluateRangeResult(BooleanLiteral.FALSE, newRanges, context.childrenResult);
@ -558,9 +573,10 @@ public class OneRangePartitionEvaluator
return context;
}
private Map<Slot, ColumnRange> replaceSlotRange(Map<Slot, ColumnRange> originRange, Slot slot, ColumnRange range) {
LinkedHashMap<Slot, ColumnRange> newRanges = Maps.newLinkedHashMap(originRange);
newRanges.put(slot, range);
private Map<Expression, ColumnRange> replaceExprRange(Map<Expression, ColumnRange> originRange, Expression expr,
ColumnRange range) {
LinkedHashMap<Expression, ColumnRange> newRanges = Maps.newLinkedHashMap(originRange);
newRanges.put(expr, range);
return ImmutableMap.copyOf(newRanges);
}
@ -568,20 +584,19 @@ public class OneRangePartitionEvaluator
Expression originResult, EvaluateRangeResult left, EvaluateRangeResult right,
BiFunction<ColumnRange, ColumnRange, ColumnRange> mergeFunction) {
Map<Slot, ColumnRange> leftRanges = left.columnRanges;
Map<Slot, ColumnRange> rightRanges = right.columnRanges;
Map<Expression, ColumnRange> leftRanges = left.columnRanges;
Map<Expression, ColumnRange> rightRanges = right.columnRanges;
if (leftRanges.equals(rightRanges)) {
return new EvaluateRangeResult(originResult, leftRanges, ImmutableList.of(left, right));
}
Set<Slot> slots = ImmutableSet.<Slot>builder()
Set<Expression> exprs = ImmutableSet.<Expression>builder()
.addAll(leftRanges.keySet())
.addAll(rightRanges.keySet())
.build();
Map<Slot, ColumnRange> mergedRange = slots.stream()
.map(slot -> Pair.of(slot, mergeFunction.apply(leftRanges.get(slot), rightRanges.get(slot))))
Map<Expression, ColumnRange> mergedRange = exprs.stream()
.map(expr -> Pair.of(expr, mergeFunction.apply(leftRanges.get(expr), rightRanges.get(expr))))
.collect(ImmutableMap.toImmutableMap(Pair::key, Pair::value));
return new EvaluateRangeResult(originResult, mergedRange, ImmutableList.of(left, right));
}
@ -616,6 +631,19 @@ public class OneRangePartitionEvaluator
return literals;
}
@Override
public EvaluateRangeResult visitDateTrunc(DateTrunc dateTrunc, EvaluateRangeInput context) {
EvaluateRangeResult result = super.visitDateTrunc(dateTrunc, context);
if (!(result.result instanceof DateTrunc)) {
return result;
}
Expression dateTruncChild = dateTrunc.child(0);
if (partitionSlotContainsNull.containsKey(dateTruncChild)) {
partitionSlotContainsNull.put(dateTrunc, true);
}
return computeMonotonicFunctionRange(result);
}
@Override
public EvaluateRangeResult visitDate(Date date, EvaluateRangeInput context) {
EvaluateRangeResult result = super.visitDate(date, context);
@ -665,13 +693,13 @@ public class OneRangePartitionEvaluator
private Map<Slot, PartitionSlotInput> fillSlotRangesToInputs(
Map<Slot, PartitionSlotInput> inputs) {
Builder<Slot, ColumnRange> allColumnRangesBuilder =
Builder<Expression, ColumnRange> allColumnRangesBuilder =
ImmutableMap.builderWithExpectedSize(16);
for (Entry<Slot, PartitionSlotInput> entry : inputs.entrySet()) {
allColumnRangesBuilder.put(entry.getKey(), entry.getValue().columnRanges.get(entry.getKey()));
}
Map<Slot, ColumnRange> allColumnRanges = allColumnRangesBuilder.build();
Map<Expression, ColumnRange> allColumnRanges = allColumnRangesBuilder.build();
Builder<Slot, PartitionSlotInput> partitionSlotInputs =
ImmutableMap.builderWithExpectedSize(16);
@ -683,12 +711,9 @@ public class OneRangePartitionEvaluator
/** EvaluateRangeInput */
public static class EvaluateRangeInput {
private Map<Slot, ColumnRange> defaultColumnRanges;
private Map<Slot, PartitionSlotInput> slotToInput;
public EvaluateRangeInput(Map<Slot, ColumnRange> defaultColumnRanges,
Map<Slot, PartitionSlotInput> slotToInput) {
this.defaultColumnRanges = defaultColumnRanges;
public EvaluateRangeInput(Map<Slot, PartitionSlotInput> slotToInput) {
this.slotToInput = slotToInput;
}
}
@ -702,7 +727,8 @@ public class OneRangePartitionEvaluator
*/
public static class EvaluateRangeResult {
private final Expression result;
private final Map<Slot, ColumnRange> columnRanges;
private final Map<Expression, ColumnRange> columnRanges;
// private final Map<Slot, ColumnRange> columnRanges;
private final List<EvaluateRangeResult> childrenResult;
// rejectNot = true, if \exist e \in R, pred(e)=true, then we have \forAll e \in R, !pred(e)=false
@ -717,7 +743,7 @@ public class OneRangePartitionEvaluator
// R=(1,10), pred: k < 11. "k<11" holds true over R, and "NOT k<11" dose not hold over R
private final boolean rejectNot;
public EvaluateRangeResult(Expression result, Map<Slot, ColumnRange> columnRanges,
public EvaluateRangeResult(Expression result, Map<Expression, ColumnRange> columnRanges,
List<EvaluateRangeResult> childrenResult, boolean rejectNot) {
this.result = result;
this.columnRanges = columnRanges;
@ -725,7 +751,7 @@ public class OneRangePartitionEvaluator
this.rejectNot = rejectNot;
}
public EvaluateRangeResult(Expression result, Map<Slot, ColumnRange> columnRanges,
public EvaluateRangeResult(Expression result, Map<Expression, ColumnRange> columnRanges,
List<EvaluateRangeResult> childrenResult) {
this(result, columnRanges, childrenResult, allIsRejectNot(childrenResult));
}
@ -757,7 +783,7 @@ public class OneRangePartitionEvaluator
Slot partitionSlot = partitionSlots.get(0);
Literal literal = (Literal) inputs.get(0).get(0);
ColumnRange slotRange = ColumnRange.singleton(literal);
ImmutableMap<Slot, ColumnRange> slotToRange = ImmutableMap.of(partitionSlot, slotRange);
ImmutableMap<Expression, ColumnRange> slotToRange = ImmutableMap.of(partitionSlot, slotRange);
Map<Slot, PartitionSlotInput> slotToInputs =
ImmutableMap.of(partitionSlot, new PartitionSlotInput(literal, slotToRange));
return ImmutableList.of(slotToInputs);
@ -810,7 +836,7 @@ public class OneRangePartitionEvaluator
previousIsLowerBoundLiteral = false;
previousIsUpperBoundLiteral = false;
}
ImmutableMap<Slot, ColumnRange> slotToRange = ImmutableMap.of(partitionSlot, slotRange);
ImmutableMap<Expression, ColumnRange> slotToRange = ImmutableMap.of(partitionSlot, slotRange);
slotToInputs.put(partitionSlot, new PartitionSlotInput(expression, slotToRange));
}
@ -819,4 +845,63 @@ public class OneRangePartitionEvaluator
}
return onePartitionInputs;
}
private EvaluateRangeResult removeEmptyRange(EvaluateRangeResult result) {
ImmutableMap.Builder<Expression, ColumnRange> builder = ImmutableMap.builder();
for (Map.Entry<Expression, ColumnRange> entry : result.columnRanges.entrySet()) {
if (entry.getValue().isEmptyRange()) {
continue;
}
builder.put(entry);
}
return new EvaluateRangeResult(result.result, builder.build(), result.childrenResult);
}
private EvaluateRangeResult computeMonotonicFunctionRange(EvaluateRangeResult result) {
Monotonic func = (Monotonic) result.result;
if (rangeMap.containsKey(func)) {
return new EvaluateRangeResult((Expression) func, ImmutableMap.of((Expression) func,
rangeMap.get(func)), result.childrenResult);
}
int childIndex = func.getMonotonicFunctionChildIndex();
Expression funcChild = func.child(childIndex);
if (!result.childrenResult.get(0).columnRanges.containsKey(funcChild)) {
return result;
}
ColumnRange childRange = result.childrenResult.get(0).columnRanges.get(funcChild);
if (childRange.isEmptyRange() || childRange.asRanges().size() != 1
|| (!childRange.span().hasLowerBound() && !childRange.span().hasUpperBound())) {
return result;
}
Range<ColumnBound> span = childRange.span();
Literal lower = span.hasLowerBound() ? span.lowerEndpoint().getValue() : null;
Literal upper = span.hasUpperBound() ? span.upperEndpoint().getValue() : null;
Expression lowerValue = lower != null ? FoldConstantRuleOnFE.evaluate(func.withConstantArgs(lower),
expressionRewriteContext) : null;
Expression upperValue = upper != null ? FoldConstantRuleOnFE.evaluate(func.withConstantArgs(upper),
expressionRewriteContext) : null;
if (!func.isPositive()) {
Expression temp = lowerValue;
lowerValue = upperValue;
upperValue = temp;
}
LinkedHashMap<Expression, ColumnRange> newRanges = Maps.newLinkedHashMap();
ColumnRange newRange = ColumnRange.all();
if (lowerValue instanceof Literal && upperValue instanceof Literal && lowerValue.equals(upperValue)) {
newRange = ColumnRange.singleton((Literal) lowerValue);
rangeMap.put((Expression) func, newRange);
newRanges.put((Expression) func, newRange);
return new EvaluateRangeResult(lowerValue, newRanges, result.childrenResult);
} else {
if (lowerValue instanceof Literal) {
newRange = newRange.withLowerBound((Literal) lowerValue);
}
if (upperValue instanceof Literal) {
newRange = newRange.withUpperBound((Literal) upperValue);
}
rangeMap.put((Expression) func, newRange);
newRanges.put((Expression) func, newRange);
return new EvaluateRangeResult((Expression) func, newRanges, result.childrenResult);
}
}
}

5
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionSlotInput.java
View File

@ -18,7 +18,6 @@
package org.apache.doris.nereids.rules.expression.rules;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import com.google.common.collect.ImmutableMap;
@ -116,9 +115,9 @@ public class PartitionSlotInput {
// part_column1 IntegerLiteral(100)
//
// because we can't fold this predicate to BooleanLiteral.FALSE, so we should scan the partition.
public final Map<Slot, ColumnRange> columnRanges;
public final Map<Expression, ColumnRange> columnRanges;
public PartitionSlotInput(Expression result, Map<Slot, ColumnRange> columnRanges) {
public PartitionSlotInput(Expression result, Map<Expression, ColumnRange> columnRanges) {
this.result = result;
this.columnRanges = ImmutableMap.copyOf(columnRanges);
}

36
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Monotonic.java Normal file
View File

@ -0,0 +1,36 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.expressions.functions;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
/** monotonicity of expressions */
public interface Monotonic extends ExpressionTrait {
// true means that the function is an increasing function
boolean isPositive();
// return the range input child index
// e.g. date_trunc(dt,'xxx') return 0
int getMonotonicFunctionChildIndex();
// return the function with the arguments replaced by literal
// e.g. date_trunc(dt, 'day'), dt in range ['2020-01-01 10:00:00', '2020-01-03 10:00:00']
// return date_trunc('2020-01-01 10:00:00', 'day')
Expression withConstantArgs(Literal literal);
}

19
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateTrunc.java
View File

@ -22,6 +22,8 @@ import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable;
import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.Monotonic;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@ -41,7 +43,7 @@ import java.util.List;
* ScalarFunction 'date_trunc'. This class is generated by GenerateFunction.
*/
public class DateTrunc extends ScalarFunction
implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable {
implements BinaryExpression, ExplicitlyCastableSignature, AlwaysNullable, Monotonic {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT)
@ -91,4 +93,19 @@ public class DateTrunc extends ScalarFunction
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitDateTrunc(this, context);
}
@Override
public boolean isPositive() {
return true;
}
@Override
public int getMonotonicFunctionChildIndex() {
return 0;
}
@Override
public Expression withConstantArgs(Literal literal) {
return new DateTrunc(literal, child(1));
}
}