[fix](Nereids) default partition be prunned by mistake (#24186)

```sql
CREATE TABLE IF NOT EXISTS t ( 
            k1 tinyint NOT NULL, 
            k2 smallint NOT NULL, 
            k3 int NOT NULL, 
            k4 bigint NOT NULL, 
            k5 decimal(9, 3) NOT NULL,
            k8 double max NOT NULL, 
            k9 float sum NOT NULL ) 
        AGGREGATE KEY(k1,k2,k3,k4,k5)
        PARTITION BY LIST(k1) ( 
            PARTITION p1 VALUES IN ("1","2","3","4"), 
            PARTITION p2 VALUES IN ("5","6","7","8"), 
            PARTITION p3 ) 
        DISTRIBUTED BY HASH(k1) BUCKETS 5 properties("replication_num" = "1")

select * from t where k1=10
```
The query will return 0 rows because p3 is pruned, we fix it by skip prune default partitions.

TODO: prune default partition if filter do not hit it
This commit is contained in:
mch_ucchi
2023-09-13 12:04:20 +08:00
committed by GitHub
parent a6f05e89f5
commit f985b28ac6
8 changed files with 55 additions and 1 deletions

View File

@ -60,4 +60,9 @@ public class HiveDefaultPartitionEvaluator implements OnePartitionEvaluator {
public Expression evaluate(Expression expression, Map<Slot, PartitionSlotInput> currentInputs) {
return BooleanLiteral.TRUE;
}
@Override
public boolean isDefaultPartition() {
return true;
}
}

View File

@ -98,4 +98,9 @@ public class OneListPartitionEvaluator
public Expression evaluate(Expression expression, Map<Slot, PartitionSlotInput> currentInputs) {
return expression.accept(this, currentInputs);
}
@Override
public boolean isDefaultPartition() {
return partitionItem.isDefaultPartition();
}
}

View File

@ -19,6 +19,7 @@ package org.apache.doris.nereids.rules.expression.rules;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import java.util.List;
import java.util.Map;
@ -45,4 +46,13 @@ public interface OnePartitionEvaluator {
* we will return a context which result expression is BooleanLiteral.FALSE
*/
Expression evaluate(Expression expression, Map<Slot, PartitionSlotInput> currentInputs);
default Expression evaluateWithDefaultPartition(Expression expression, Map<Slot, PartitionSlotInput> inputs) {
if (isDefaultPartition()) {
return BooleanLiteral.TRUE;
}
return evaluate(expression, inputs);
}
boolean isDefaultPartition();
}

View File

@ -665,4 +665,9 @@ public class OneRangePartitionEvaluator
this.childrenResult = childrenResult;
}
}
@Override
public boolean isDefaultPartition() {
return partitionItem.isDefaultPartition();
}
}

View File

@ -83,6 +83,7 @@ public class PartitionPruner {
.collect(ImmutableList.toImmutableList());
PartitionPruner partitionPruner = new PartitionPruner(evaluators, partitionPredicate);
//TODO: we keep default partition because it's too hard to prune it, we return false in canPrune().
return partitionPruner.prune();
}
@ -110,7 +111,7 @@ public class PartitionPruner {
private boolean canPrune(OnePartitionEvaluator evaluator) {
List<Map<Slot, PartitionSlotInput>> onePartitionInputs = evaluator.getOnePartitionInputs();
for (Map<Slot, PartitionSlotInput> currentInputs : onePartitionInputs) {
Expression result = evaluator.evaluate(partitionPredicate, currentInputs);
Expression result = evaluator.evaluateWithDefaultPartition(partitionPredicate, currentInputs);
if (!result.equals(BooleanLiteral.FALSE)) {
return false;
}

View File

@ -52,4 +52,9 @@ public class UnknownPartitionEvaluator implements OnePartitionEvaluator {
// do not prune
return expression;
}
@Override
public boolean isDefaultPartition() {
return partitionItem.isDefaultPartition();
}
}

View File

@ -68,6 +68,7 @@ public class PruneOlapScanPartition extends OneRewriteRuleFactory {
List<Long> prunedPartitions = new ArrayList<>(PartitionPruner.prune(
partitionSlots, filter.getPredicate(), partitionInfo, ctx.cascadesContext,
PartitionTableType.OLAP));
List<Long> manuallySpecifiedPartitions = scan.getManuallySpecifiedPartitions();
if (!CollectionUtils.isEmpty(manuallySpecifiedPartitions)) {
prunedPartitions.retainAll(manuallySpecifiedPartitions);

View File

@ -196,6 +196,28 @@ class PruneOlapScanPartitionTest extends TestWithFeService implements MemoPatter
test("testOlapScanPartitionPruneWithMultiColumnCase", "cast(col1 as bigint) + 1 = 5", 1);
}
@Test
public void prunePartitionWithDefaultPartition() throws Exception {
createTable("CREATE TABLE IF NOT EXISTS test_default_in_parts (\n"
+ " k1 tinyint NOT NULL, \n"
+ " k2 smallint NOT NULL, \n"
+ " k3 int NOT NULL, \n"
+ " k4 bigint NOT NULL, \n"
+ " k5 decimal(9, 3) NOT NULL,\n"
+ " k8 double max NOT NULL, \n"
+ " k9 float sum NOT NULL ) \n"
+ " AGGREGATE KEY(k1,k2,k3,k4,k5)\n"
+ " PARTITION BY LIST(k1) ( \n"
+ " PARTITION p1 VALUES IN (\"1\",\"2\",\"3\",\"4\"), \n"
+ " PARTITION p2 VALUES IN (\"5\",\"6\",\"7\",\"8\"), \n"
+ " PARTITION p3 ) \n"
+ " DISTRIBUTED BY HASH(k1) BUCKETS 5 properties(\"replication_num\" = \"1\")");
test("test_default_in_parts", "k1 = 10", 1);
test("test_default_in_parts", "k1 > 5", 2);
test("test_default_in_parts", "k1 > 2", 3);
test("test_default_in_parts", "(k1 > 1 and k1 < 8)", 3);
}
@Test
public void prunePartitionWithOrPredicate() {
test("test_list_parts", "(part = 9 and id <= 500) or (part = 3)", 1);