database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[refactor](planner) filter empty partitions in a unified location (#27190)

Browse Source
This commit is contained in:
walter
2023-11-17 23:58:21 +08:00
committed by GitHub
parent 5fb27eb652
commit 9319d173dd
3 changed files with 15 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View File

@ -965,6 +965,18 @@ public class OlapTable extends Table {
return partition;
}
// select the non-empty partition ids belonging to this table.
//
// ATTN: partitions not belonging to this table will be filtered.
public List<Long> selectNonEmptyPartitionIds(Collection<Long> partitionIds) {
return partitionIds.stream()
.map(this::getPartition)
.filter(p -> p != null)
.filter(Partition::hasData)
.map(Partition::getId)
.collect(Collectors.toList());
}
public int getPartitionNum() {
return idToPartition.size();
}

6
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java
View File

@ -22,8 +22,6 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import java.util.stream.Collectors;
/**
* Used to prune empty partition.
*/
@ -34,9 +32,7 @@ public class PruneEmptyPartition extends OneRewriteRuleFactory {
return logicalOlapScan().thenApply(ctx -> {
LogicalOlapScan scan = ctx.root;
OlapTable table = scan.getTable();
return scan.withSelectedPartitionIds(scan.getSelectedPartitionIds().stream()
.filter(partitionId -> table.getPartition(partitionId).hasData())
.collect(Collectors.toList()));
return scan.withSelectedPartitionIds(table.selectNonEmptyPartitionIds(scan.getSelectedPartitionIds()));
}).toRule(RuleType.PRUNE_EMPTY_PARTITION);
}
}

17
fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
View File

@ -873,22 +873,9 @@ public class OlapScanNode extends ScanNode {
if (partitionInfo.getType() == PartitionType.RANGE || partitionInfo.getType() == PartitionType.LIST) {
selectedPartitionIds = partitionPrune(partitionInfo, partitionNames);
} else {
selectedPartitionIds = null;
}
if (selectedPartitionIds == null) {
selectedPartitionIds = Lists.newArrayList();
for (Partition partition : olapTable.getPartitions()) {
if (!partition.hasData()) {
continue;
}
selectedPartitionIds.add(partition.getId());
}
} else {
selectedPartitionIds = selectedPartitionIds.stream()
.filter(id -> olapTable.getPartition(id).hasData())
.collect(Collectors.toList());
selectedPartitionIds = olapTable.getPartitionIds();
}
selectedPartitionIds = olapTable.selectNonEmptyPartitionIds(selectedPartitionIds);
selectedPartitionNum = selectedPartitionIds.size();
for (long id : selectedPartitionIds) {