database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

branch-2.1: [fix](nereids)Solve the problem of pruning wrong partitions in multi-column partition pruning (#43658)

Browse Source 
Cherry-picked from #43332

Co-authored-by: feiniaofeiafei <53502832+feiniaofeiafei@users.noreply.github.com>
This commit is contained in:
github-actions[bot]
2024-11-12 19:14:51 +08:00
committed by GitHub
parent 797a31bbb8
commit 7b3357477d
2 changed files with 41 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
View File

@ -93,7 +93,6 @@ public class OneRangePartitionEvaluator
private final List<List<Expression>> inputs;
private final Map<Expression, Boolean> partitionSlotContainsNull;
private final Map<Slot, PartitionSlotType> slotToType;
private final Map<Expression, ColumnRange> rangeMap = new HashMap<>();
/** OneRangePartitionEvaluator */
public OneRangePartitionEvaluator(long partitionId, List<Slot> partitionSlots,
@ -175,8 +174,8 @@ public class OneRangePartitionEvaluator
@Override
public Expression evaluate(Expression expression, Map<Slot, PartitionSlotInput> currentInputs) {
Map<Expression, ColumnRange> defaultColumnRanges = currentInputs.values().iterator().next().columnRanges;
rangeMap.putAll(defaultColumnRanges);
EvaluateRangeResult result = expression.accept(this, new EvaluateRangeInput(currentInputs));
Map<Expression, ColumnRange> rangeMap = new HashMap<>(defaultColumnRanges);
EvaluateRangeResult result = expression.accept(this, new EvaluateRangeInput(currentInputs, rangeMap));
return result.result;
}
@ -397,6 +396,7 @@ public class OneRangePartitionEvaluator
public EvaluateRangeResult visitAnd(And and, EvaluateRangeInput context) {
EvaluateRangeResult result = evaluateChildrenThenThis(and, context);
result = mergeRanges(result.result, result.childrenResult.get(0), result.childrenResult.get(1),
context.rangeMap,
(leftRange, rightRange) -> leftRange.intersect(rightRange));
result = returnFalseIfExistEmptyRange(result);
@ -425,6 +425,7 @@ public class OneRangePartitionEvaluator
result.childrenResult);
}
result = mergeRanges(result.result, result.childrenResult.get(0), result.childrenResult.get(1),
context.rangeMap,
(leftRange, rightRange) -> leftRange.union(rightRange));
return returnFalseIfExistEmptyRange(result);
}
@ -437,8 +438,8 @@ public class OneRangePartitionEvaluator
for (Map.Entry<Expression, ColumnRange> entry : result.childrenResult.get(0).columnRanges.entrySet()) {
Expression expr = entry.getKey();
ColumnRange childRange = entry.getValue();
ColumnRange partitionRange = rangeMap.containsKey(expr)
? rangeMap.get(expr) : ColumnRange.all();
ColumnRange partitionRange = context.rangeMap.containsKey(expr)
? context.rangeMap.get(expr) : ColumnRange.all();
newRanges.put(expr, partitionRange.intersect(childRange.complete()));
}
result = new EvaluateRangeResult(result.result, newRanges, result.childrenResult);
@ -562,6 +563,7 @@ public class OneRangePartitionEvaluator
private EvaluateRangeResult mergeRanges(
Expression originResult, EvaluateRangeResult left, EvaluateRangeResult right,
Map<Expression, ColumnRange> rangeMap,
BiFunction<ColumnRange, ColumnRange, ColumnRange> mergeFunction) {
Map<Expression, ColumnRange> leftRanges = left.columnRanges;
@ -623,7 +625,7 @@ public class OneRangePartitionEvaluator
if (partitionSlotContainsNull.containsKey(dateTruncChild)) {
partitionSlotContainsNull.put(dateTrunc, true);
}
return computeMonotonicFunctionRange(result);
return computeMonotonicFunctionRange(result, context.rangeMap);
}
@Override
@ -636,7 +638,7 @@ public class OneRangePartitionEvaluator
if (partitionSlotContainsNull.containsKey(dateChild)) {
partitionSlotContainsNull.put(date, true);
}
return computeMonotonicFunctionRange(result);
return computeMonotonicFunctionRange(result, context.rangeMap);
}
@Override
@ -649,7 +651,7 @@ public class OneRangePartitionEvaluator
if (partitionSlotContainsNull.containsKey(converTzChild)) {
partitionSlotContainsNull.put(convertTz, true);
}
return computeMonotonicFunctionRange(result);
return computeMonotonicFunctionRange(result, context.rangeMap);
}
private boolean isPartitionSlot(Slot slot) {
@ -681,10 +683,12 @@ public class OneRangePartitionEvaluator
/** EvaluateRangeInput */
public static class EvaluateRangeInput {
private Map<Slot, PartitionSlotInput> slotToInput;
private final Map<Slot, PartitionSlotInput> slotToInput;
private final Map<Expression, ColumnRange> rangeMap;
public EvaluateRangeInput(Map<Slot, PartitionSlotInput> slotToInput) {
public EvaluateRangeInput(Map<Slot, PartitionSlotInput> slotToInput, Map<Expression, ColumnRange> rangeMap) {
this.slotToInput = slotToInput;
this.rangeMap = rangeMap;
}
}
@ -816,7 +820,8 @@ public class OneRangePartitionEvaluator
return onePartitionInputs;
}
private EvaluateRangeResult computeMonotonicFunctionRange(EvaluateRangeResult result) {
private EvaluateRangeResult computeMonotonicFunctionRange(EvaluateRangeResult result,
Map<Expression, ColumnRange> rangeMap) {
Monotonic func = (Monotonic) result.result;
if (rangeMap.containsKey(func)) {
return new EvaluateRangeResult((Expression) func, ImmutableMap.of((Expression) func,

25
regression-test/suites/nereids_rules_p0/partition_prune/test_date_trunc_prune.groovy
View File

@ -334,4 +334,29 @@ suite("test_date_trunc_prune") {
( NOT ( ( table1 . `col_int_undef_signed` != table1 . `col_int_undef_signed` ) AND table1 . `col_date_undef_signed` <= '2025-06-18' ) AND
table1 . `col_date_undef_signed` IN ( '2023-12-20' ) ) GROUP BY field1 ORDER BY field1 LIMIT 1000;
"""
// test multi column partition table and predicate with date_trunc
sql "drop table if exists t_multi_column_partition"
sql """
create table t_multi_column_partition(a int, dt datetime, v int) partition by range(a, dt)
(
partition p0 values [(0,'2024-01-01 00:00:00'), (10,'2024-01-10 00:00:00')),
partition p10 values [(10,'2024-01-10 00:00:00'), (20,'2024-01-20 00:00:00')),
partition p20 values [(20,'2024-01-20 00:00:00'), (30,'2024-01-31 00:00:00')),
partition p30 values [(30,'2024-01-31 00:00:00'), (40,'2024-02-10 00:00:00')),
partition p40 values [(40,'2024-02-10 00:00:00'), (50,'2024-02-20 00:00:00'))
)
distributed by hash(a) properties("replication_num"="1");
"""
sql """
insert into t_multi_column_partition values(0,'2024-01-01 00:00:00',2),(1,'2024-01-01 00:00:00',2),(1,'2025-01-01 00:00:00',2),
(10,'2024-01-10 00:00:00',3),(10,'2024-01-11 00:00:00',200),(12,'2021-01-01 00:00:00',2),
(25,'2024-01-10 00:00:00',3),(20,'2024-01-11 00:00:00',200),(30,'2021-01-01 00:00:00',2),
(40,'2024-01-01 00:00:00',2),(40,'2024-01-31 00:00:00',2),(10,'2024-01-9 00:00:00',1000),(10,'2024-01-10 00:00:00',1000),(10,'2024-01-10 01:00:00',1000),
(2,'2023-01-10 01:00:00',1000)
"""
explain {
sql """select * from t_multi_column_partition where a=2 and date_trunc(dt,'day') <'2024-01-1 00:00:00';"""
contains ("partitions=1/5 (p0)")
}
}