[fix](nereids) partition prune fails in case of NOT expression (#27047)
* handle not and add regression test
This commit is contained in:
minghong
@ -346,13 +346,16 @@ public class OneRangePartitionEvaluator
|
||||
if (!(result.result instanceof EqualTo)) {
|
||||
return result;
|
||||
}
|
||||
equalTo = (EqualTo) result.result;
|
||||
boolean isRejectNot = false;
|
||||
if (equalTo.left() instanceof Slot && equalTo.right() instanceof Literal) {
|
||||
Slot slot = (Slot) equalTo.left();
|
||||
if (isPartitionSlot(slot)) {
|
||||
Map<Slot, ColumnRange> leftColumnRanges = result.childrenResult.get(0).columnRanges;
|
||||
ColumnRange atLeastRange = ColumnRange.singleton((Literal) equalTo.right());
|
||||
result = intersectSlotRange(result, leftColumnRanges, slot, atLeastRange);
|
||||
if (leftColumnRanges.get(slot).isSingleton()) {
|
||||
isRejectNot = true;
|
||||
}
|
||||
}
|
||||
} else if (equalTo.left() instanceof Literal && equalTo.right() instanceof Slot) {
|
||||
Slot slot = (Slot) equalTo.right();
|
||||
@ -360,7 +363,15 @@ public class OneRangePartitionEvaluator
|
||||
Map<Slot, ColumnRange> rightColumnRanges = result.childrenResult.get(1).columnRanges;
|
||||
ColumnRange atMostRange = ColumnRange.singleton((Literal) equalTo.left());
|
||||
result = intersectSlotRange(result, rightColumnRanges, slot, atMostRange);
|
||||
if (rightColumnRanges.get(slot).isSingleton()) {
|
||||
isRejectNot = true;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
isRejectNot = false;
|
||||
}
|
||||
if (!isRejectNot) {
|
||||
result = result.withRejectNot(false);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@ -383,6 +394,7 @@ public class OneRangePartitionEvaluator
|
||||
Map<Slot, ColumnRange> slotRanges = result.childrenResult.get(0).columnRanges;
|
||||
result = intersectSlotRange(result, slotRanges, slot, unionLiteralRange);
|
||||
}
|
||||
result = result.withRejectNot(false);
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -392,14 +404,15 @@ public class OneRangePartitionEvaluator
|
||||
if (!(result.result instanceof IsNull)) {
|
||||
return result;
|
||||
}
|
||||
|
||||
result = result.withRejectNot(false);
|
||||
Expression child = isNull.child();
|
||||
if (!(child instanceof Slot) || !isPartitionSlot((Slot) child)) {
|
||||
return result;
|
||||
}
|
||||
|
||||
if (!partitionSlotContainsNull.get((Slot) child)) {
|
||||
return new EvaluateRangeResult(BooleanLiteral.FALSE, result.columnRanges, result.childrenResult);
|
||||
return new EvaluateRangeResult(BooleanLiteral.FALSE,
|
||||
result.columnRanges, result.childrenResult, false);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@ -434,12 +447,16 @@ public class OneRangePartitionEvaluator
|
||||
@Override
|
||||
public EvaluateRangeResult visitNot(Not not, EvaluateRangeInput context) {
|
||||
EvaluateRangeResult result = evaluateChildrenThenThis(not, context);
|
||||
|
||||
Map<Slot, ColumnRange> newRanges = result.childrenResult.get(0).columnRanges.entrySet()
|
||||
.stream()
|
||||
.map(slotToRange -> Pair.of(slotToRange.getKey(), slotToRange.getValue().complete()))
|
||||
.collect(ImmutableMap.toImmutableMap(Pair::key, Pair::value));
|
||||
result = new EvaluateRangeResult(result.result, newRanges, result.childrenResult);
|
||||
if (result.isRejectNot()) {
|
||||
Map<Slot, ColumnRange> newRanges = Maps.newHashMap();
|
||||
for (Map.Entry<Slot, ColumnRange> entry : result.childrenResult.get(0).columnRanges.entrySet()) {
|
||||
Slot slot = entry.getKey();
|
||||
ColumnRange childRange = entry.getValue();
|
||||
ColumnRange partitionRange = result.columnRanges.get(slot);
|
||||
newRanges.put(slot, partitionRange.intersect(childRange.complete()));
|
||||
}
|
||||
result = new EvaluateRangeResult(result.result, newRanges, result.childrenResult);
|
||||
}
|
||||
return returnFalseIfExistEmptyRange(result);
|
||||
}
|
||||
|
||||
@ -662,11 +679,37 @@ public class OneRangePartitionEvaluator
|
||||
private final Map<Slot, ColumnRange> columnRanges;
|
||||
private final List<EvaluateRangeResult> childrenResult;
|
||||
|
||||
// rejectNot = true, if \exist e \in R, pred(e)=true, then we have \forAll e \in R, !pred(e)=false
|
||||
// that is, if pred holds true over R, then !pred does not hold true over R.
|
||||
// example 1. rejectNot=false
|
||||
// R=(1,10), pred: k = 5. "k = 5" holds true over R, and "NOT k = 5" holds true over R.
|
||||
// example 2. rejectNot=false
|
||||
// R=(1,10), pred: k = 11. "k=10" dose not holds over R
|
||||
// example 3. rejectNot=false
|
||||
// R=(1,10), pred: k in (4, 5). "k in (4, 5)" holds true over R, and "NOT k in (4, 5)" holds over R
|
||||
// example 3. rejectNot=true
|
||||
// R=(1,10), pred: k < 11. "k<11" holds true over R, and "NOT k<11" dose not hold over R
|
||||
private final boolean rejectNot;
|
||||
|
||||
public EvaluateRangeResult(Expression result, Map<Slot, ColumnRange> columnRanges,
|
||||
List<EvaluateRangeResult> childrenResult) {
|
||||
List<EvaluateRangeResult> childrenResult, boolean rejectNot) {
|
||||
this.result = result;
|
||||
this.columnRanges = columnRanges;
|
||||
this.childrenResult = childrenResult;
|
||||
this.rejectNot = rejectNot;
|
||||
}
|
||||
|
||||
public EvaluateRangeResult(Expression result, Map<Slot, ColumnRange> columnRanges,
|
||||
List<EvaluateRangeResult> childrenResult) {
|
||||
this(result, columnRanges, childrenResult, childrenResult.stream().allMatch(r -> r.isRejectNot()));
|
||||
}
|
||||
|
||||
public EvaluateRangeResult withRejectNot(boolean rejectNot) {
|
||||
return new EvaluateRangeResult(result, columnRanges, childrenResult, rejectNot);
|
||||
}
|
||||
|
||||
public boolean isRejectNot() {
|
||||
return rejectNot;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -75,10 +75,10 @@ suite("test_multi_range_partition") {
|
||||
contains "artitions=3/3 (p1,p2,p3)"
|
||||
}
|
||||
|
||||
// BUG: p1 missed
|
||||
// fix BUG: p1 missed
|
||||
explain{
|
||||
sql "select * from pt where sin(k1)<>0"
|
||||
contains "partitions=2/3 (p2,p3)"
|
||||
contains "partitions=3/3 (p1,p2,p3)"
|
||||
}
|
||||
|
||||
// ============= in predicate ======================
|
||||
@ -98,10 +98,10 @@ suite("test_multi_range_partition") {
|
||||
contains "partitions=1/3 (p1)"
|
||||
}
|
||||
|
||||
// BUG: p1 missed
|
||||
// fix BUG: p1 missed
|
||||
explain{
|
||||
sql "select * from pt where k1 is not null"
|
||||
contains "partitions=2/3 (p2,p3)"
|
||||
contains "partitions=3/3 (p1,p2,p3)"
|
||||
}
|
||||
|
||||
//======== the second partition key =========
|
||||
@ -126,10 +126,10 @@ suite("test_multi_range_partition") {
|
||||
contains "partitions=1/3 (p2)"
|
||||
}
|
||||
|
||||
//BUG: p2 missed
|
||||
//fix BUG: p2 missed
|
||||
explain {
|
||||
sql "select * from pt where k1=7 and k1<>k2"
|
||||
contains "partitions=1/3 (p3)"
|
||||
contains "partitions=2/3 (p2,p3)"
|
||||
}
|
||||
|
||||
//p3 NOT pruned
|
||||
@ -138,10 +138,10 @@ suite("test_multi_range_partition") {
|
||||
contains "partitions=2/3 (p2,p3)"
|
||||
}
|
||||
|
||||
//BUG: p2 missed
|
||||
//fix BUG: p2 missed
|
||||
explain {
|
||||
sql "select * from pt where k1=7 and not (k2 is null);"
|
||||
contains "VEMPTYSET"
|
||||
contains "partitions=2/3 (p2,p3)"
|
||||
}
|
||||
|
||||
//p3 NOT pruned
|
||||
@ -150,10 +150,10 @@ suite("test_multi_range_partition") {
|
||||
contains "partitions=2/3 (p2,p3)"
|
||||
}
|
||||
|
||||
//BUG: p2 missed
|
||||
//fix BUG: p2 missed
|
||||
explain {
|
||||
sql "select * from pt where k1=7 and k2 not in (1, 2);"
|
||||
contains "partitions=1/3 (p3)"
|
||||
contains "partitions=2/3 (p2,p3)"
|
||||
}
|
||||
|
||||
explain {
|
||||
@ -161,10 +161,10 @@ suite("test_multi_range_partition") {
|
||||
contains "partitions=2/3 (p2,p3)"
|
||||
}
|
||||
|
||||
//BUG: p2,p3 pruned
|
||||
//fix BUG: p2,p3 pruned
|
||||
explain {
|
||||
sql "select * from pt where k1=7 and k2 not in (1, 12)"
|
||||
contains "VEMPTYSET"
|
||||
contains "partitions=2/3 (p2,p3)"
|
||||
}
|
||||
|
||||
explain {
|
||||
@ -238,4 +238,38 @@ suite("test_multi_range_partition") {
|
||||
contains "partitions=2/3 (p2,p3)"
|
||||
}
|
||||
|
||||
|
||||
// test if a range is not sliced to multiple single point
|
||||
// for example: range [3,7) is sliced to [3,3], [4,4],[5,5],[6,6]
|
||||
sql "set partition_pruning_expand_threshold=1;"
|
||||
|
||||
explain {
|
||||
sql "select * from pt where k1 < 5;"
|
||||
contains "partitions=2/3 (p1,p2)"
|
||||
}
|
||||
|
||||
explain {
|
||||
sql "select * from pt where not k1 < 5;"
|
||||
contains "partitions=2/3 (p2,p3)"
|
||||
}
|
||||
|
||||
explain {
|
||||
sql "select * from pt where k2 < 5;"
|
||||
contains "partitions=3/3 (p1,p2,p3)"
|
||||
}
|
||||
|
||||
explain {
|
||||
sql "select * from pt where not k2 < 5;"
|
||||
contains "partitions=3/3 (p1,p2,p3)"
|
||||
}
|
||||
|
||||
explain {
|
||||
sql "select * from pt where k3 < 5;"
|
||||
contains "partitions=3/3 (p1,p2,p3)"
|
||||
}
|
||||
|
||||
explain {
|
||||
sql "select * from pt where not k3 < 5;"
|
||||
contains "partitions=3/3 (p1,p2,p3)"
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user