database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](nereids) temporary partition is selected only if user manually specified (#27893)

Browse Source 
q1: "select * from ut_p temporary partitions(tp1) where val > 0"
in q1, temporary partition tp1 is scaned

q2: "select * from ut_p where val > 0"
in q2, temporary partition tp1 is not scaned.
This commit is contained in:
minghong
2023-12-04 09:44:27 +08:00
committed by GitHub
parent dfad778d51
commit f2cfc87aca
4 changed files with 42 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
View File

@ -18,7 +18,6 @@
package org.apache.doris.nereids.rules.expression.rules;
import org.apache.doris.catalog.ListPartitionItem;
import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.RangePartitionItem;
import org.apache.doris.nereids.CascadesContext;
@ -98,15 +97,6 @@ public class PartitionPruner extends DefaultExpressionRewriter<Void> {
.collect(ImmutableList.toImmutableList());
}
/**
* prune partition with `partitionInfo` as parameter.
*/
public static List<Long> prune(List<Slot> partitionSlots, Expression partitionPredicate,
PartitionInfo partitionInfo, CascadesContext cascadesContext, PartitionTableType partitionTableType) {
return prune(partitionSlots, partitionPredicate, partitionInfo.getAllPartitions(), cascadesContext,
partitionTableType);
}
/**
* prune partition with `idToPartitions` as parameter.
*/

18
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java
View File

@ -19,6 +19,7 @@ package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.expression.rules.PartitionPruner;
@ -31,7 +32,6 @@ import org.apache.doris.nereids.util.Utils;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
import org.apache.commons.collections.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
@ -66,15 +66,21 @@ public class PruneOlapScanPartition extends OneRewriteRuleFactory {
.stream()
.map(column -> scanOutput.get(column.getName().toLowerCase()))
.collect(Collectors.toList());
List<Long> manuallySpecifiedPartitions = scan.getManuallySpecifiedPartitions();
Map<Long, PartitionItem> idToPartitions;
if (manuallySpecifiedPartitions.isEmpty()) {
idToPartitions = partitionInfo.getIdToItem(false);
} else {
Map<Long, PartitionItem> allPartitions = partitionInfo.getAllPartitions();
idToPartitions = allPartitions.keySet().stream()
.filter(id -> manuallySpecifiedPartitions.contains(id))
.collect(Collectors.toMap(Function.identity(), id -> allPartitions.get(id)));
}
List<Long> prunedPartitions = new ArrayList<>(PartitionPruner.prune(
partitionSlots, filter.getPredicate(), partitionInfo, ctx.cascadesContext,
partitionSlots, filter.getPredicate(), idToPartitions, ctx.cascadesContext,
PartitionTableType.OLAP));
List<Long> manuallySpecifiedPartitions = scan.getManuallySpecifiedPartitions();
if (!CollectionUtils.isEmpty(manuallySpecifiedPartitions)) {
prunedPartitions.retainAll(manuallySpecifiedPartitions);
}
if (prunedPartitions.isEmpty()) {
return new LogicalEmptyRelation(
ConnectContext.get().getStatementContext().getNextRelationId(),

2
regression-test/data/nereids_syntax_p0/select_partition.out
View File

@ -25,5 +25,5 @@
1 aaa aaa
-- !sql --
16 1234 t
6 1234 t

32
regression-test/suites/nereids_syntax_p0/select_partition.groovy
View File

@ -89,6 +89,7 @@ suite("query_on_specific_partition") {
SELECT * FROM test_iot PARTITION p1;
"""
// temporary partition test
sql """
DROP TABLE IF EXISTS ut_p;
"""
@ -110,14 +111,39 @@ suite("query_on_specific_partition") {
);
"""
sql """ALTER TABLE ut_p ADD TEMPORARY PARTITION tp1 VALUES [("15"), ("20"));"""
sql """ALTER TABLE ut_p ADD TEMPORARY PARTITION tp1 VALUES [("5"), ("7"));"""
sql "INSERT INTO ut_p TEMPORARY PARTITION(tp1) values(16,1234, 't');"
sql "INSERT INTO ut_p TEMPORARY PARTITION(tp1) values(6,1234, 't');"
sql "INSERT INTO ut_p values(6,1234, 't');"
sql "INSERT INTO ut_p values(3,1234, 't');"
sql "set enable_nereids_planner=true"
sql "SET enable_fallback_to_original_planner=false"
qt_sql """select * from ut_p temporary partitions(tp1);"""
explain {
sql "select * from ut_p temporary partitions(tp1);"
contains "partitions=1/2 (tp1)"
}
explain {
sql "select * from ut_p temporary partitions(tp1) where val > 0"
contains "partitions=1/2 (tp1)"
}
explain {
sql "select * from ut_p temporary partitions(tp1) where val > 0"
contains "partitions=1/2 (tp1)"
}
explain {
sql "select * from ut_p partitions(p2) where val > 0"
contains "partitions=1/2 (p2)"
}
explain {
sql "select * from ut_p temporary partitions(tp1) where id = 8"
contains "VEMPTYSET"
}
}