[fix](Nereids) default partition be prunned by mistake (#24186)
```sql
CREATE TABLE IF NOT EXISTS t (
k1 tinyint NOT NULL,
k2 smallint NOT NULL,
k3 int NOT NULL,
k4 bigint NOT NULL,
k5 decimal(9, 3) NOT NULL,
k8 double max NOT NULL,
k9 float sum NOT NULL )
AGGREGATE KEY(k1,k2,k3,k4,k5)
PARTITION BY LIST(k1) (
PARTITION p1 VALUES IN ("1","2","3","4"),
PARTITION p2 VALUES IN ("5","6","7","8"),
PARTITION p3 )
DISTRIBUTED BY HASH(k1) BUCKETS 5 properties("replication_num" = "1")
select * from t where k1=10
```
The query will return 0 rows because p3 is pruned, we fix it by skip prune default partitions.
TODO: prune default partition if filter do not hit it
This commit is contained in:
@ -60,4 +60,9 @@ public class HiveDefaultPartitionEvaluator implements OnePartitionEvaluator {
|
||||
public Expression evaluate(Expression expression, Map<Slot, PartitionSlotInput> currentInputs) {
|
||||
return BooleanLiteral.TRUE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isDefaultPartition() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
@ -98,4 +98,9 @@ public class OneListPartitionEvaluator
|
||||
public Expression evaluate(Expression expression, Map<Slot, PartitionSlotInput> currentInputs) {
|
||||
return expression.accept(this, currentInputs);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isDefaultPartition() {
|
||||
return partitionItem.isDefaultPartition();
|
||||
}
|
||||
}
|
||||
|
||||
@ -19,6 +19,7 @@ package org.apache.doris.nereids.rules.expression.rules;
|
||||
|
||||
import org.apache.doris.nereids.trees.expressions.Expression;
|
||||
import org.apache.doris.nereids.trees.expressions.Slot;
|
||||
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
@ -45,4 +46,13 @@ public interface OnePartitionEvaluator {
|
||||
* we will return a context which result expression is BooleanLiteral.FALSE
|
||||
*/
|
||||
Expression evaluate(Expression expression, Map<Slot, PartitionSlotInput> currentInputs);
|
||||
|
||||
default Expression evaluateWithDefaultPartition(Expression expression, Map<Slot, PartitionSlotInput> inputs) {
|
||||
if (isDefaultPartition()) {
|
||||
return BooleanLiteral.TRUE;
|
||||
}
|
||||
return evaluate(expression, inputs);
|
||||
}
|
||||
|
||||
boolean isDefaultPartition();
|
||||
}
|
||||
|
||||
@ -665,4 +665,9 @@ public class OneRangePartitionEvaluator
|
||||
this.childrenResult = childrenResult;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isDefaultPartition() {
|
||||
return partitionItem.isDefaultPartition();
|
||||
}
|
||||
}
|
||||
|
||||
@ -83,6 +83,7 @@ public class PartitionPruner {
|
||||
.collect(ImmutableList.toImmutableList());
|
||||
|
||||
PartitionPruner partitionPruner = new PartitionPruner(evaluators, partitionPredicate);
|
||||
//TODO: we keep default partition because it's too hard to prune it, we return false in canPrune().
|
||||
return partitionPruner.prune();
|
||||
}
|
||||
|
||||
@ -110,7 +111,7 @@ public class PartitionPruner {
|
||||
private boolean canPrune(OnePartitionEvaluator evaluator) {
|
||||
List<Map<Slot, PartitionSlotInput>> onePartitionInputs = evaluator.getOnePartitionInputs();
|
||||
for (Map<Slot, PartitionSlotInput> currentInputs : onePartitionInputs) {
|
||||
Expression result = evaluator.evaluate(partitionPredicate, currentInputs);
|
||||
Expression result = evaluator.evaluateWithDefaultPartition(partitionPredicate, currentInputs);
|
||||
if (!result.equals(BooleanLiteral.FALSE)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -52,4 +52,9 @@ public class UnknownPartitionEvaluator implements OnePartitionEvaluator {
|
||||
// do not prune
|
||||
return expression;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isDefaultPartition() {
|
||||
return partitionItem.isDefaultPartition();
|
||||
}
|
||||
}
|
||||
|
||||
@ -68,6 +68,7 @@ public class PruneOlapScanPartition extends OneRewriteRuleFactory {
|
||||
List<Long> prunedPartitions = new ArrayList<>(PartitionPruner.prune(
|
||||
partitionSlots, filter.getPredicate(), partitionInfo, ctx.cascadesContext,
|
||||
PartitionTableType.OLAP));
|
||||
|
||||
List<Long> manuallySpecifiedPartitions = scan.getManuallySpecifiedPartitions();
|
||||
if (!CollectionUtils.isEmpty(manuallySpecifiedPartitions)) {
|
||||
prunedPartitions.retainAll(manuallySpecifiedPartitions);
|
||||
|
||||
@ -196,6 +196,28 @@ class PruneOlapScanPartitionTest extends TestWithFeService implements MemoPatter
|
||||
test("testOlapScanPartitionPruneWithMultiColumnCase", "cast(col1 as bigint) + 1 = 5", 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void prunePartitionWithDefaultPartition() throws Exception {
|
||||
createTable("CREATE TABLE IF NOT EXISTS test_default_in_parts (\n"
|
||||
+ " k1 tinyint NOT NULL, \n"
|
||||
+ " k2 smallint NOT NULL, \n"
|
||||
+ " k3 int NOT NULL, \n"
|
||||
+ " k4 bigint NOT NULL, \n"
|
||||
+ " k5 decimal(9, 3) NOT NULL,\n"
|
||||
+ " k8 double max NOT NULL, \n"
|
||||
+ " k9 float sum NOT NULL ) \n"
|
||||
+ " AGGREGATE KEY(k1,k2,k3,k4,k5)\n"
|
||||
+ " PARTITION BY LIST(k1) ( \n"
|
||||
+ " PARTITION p1 VALUES IN (\"1\",\"2\",\"3\",\"4\"), \n"
|
||||
+ " PARTITION p2 VALUES IN (\"5\",\"6\",\"7\",\"8\"), \n"
|
||||
+ " PARTITION p3 ) \n"
|
||||
+ " DISTRIBUTED BY HASH(k1) BUCKETS 5 properties(\"replication_num\" = \"1\")");
|
||||
test("test_default_in_parts", "k1 = 10", 1);
|
||||
test("test_default_in_parts", "k1 > 5", 2);
|
||||
test("test_default_in_parts", "k1 > 2", 3);
|
||||
test("test_default_in_parts", "(k1 > 1 and k1 < 8)", 3);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void prunePartitionWithOrPredicate() {
|
||||
test("test_list_parts", "(part = 9 and id <= 500) or (part = 3)", 1);
|
||||
|
||||
Reference in New Issue
Block a user