database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](Nereids) Filter no data partition after partition prune (#25456)

Browse Source
This commit is contained in:
AKIRA
2023-10-25 17:37:10 +08:00
committed by GitHub
parent 8a436d8ecc
commit 20fa1eff65
5 changed files with 45 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
View File

@ -80,6 +80,7 @@ import org.apache.doris.nereids.rules.rewrite.MergeOneRowRelationIntoUnion;
import org.apache.doris.nereids.rules.rewrite.MergeProjects;
import org.apache.doris.nereids.rules.rewrite.MergeSetOperations;
import org.apache.doris.nereids.rules.rewrite.NormalizeSort;
import org.apache.doris.nereids.rules.rewrite.PruneEmptyPartition;
import org.apache.doris.nereids.rules.rewrite.PruneFileScanPartition;
import org.apache.doris.nereids.rules.rewrite.PruneOlapScanPartition;
import org.apache.doris.nereids.rules.rewrite.PruneOlapScanTablet;
@ -300,6 +301,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
topic("Table/Physical optimization",
topDown(
new PruneOlapScanPartition(),
new PruneEmptyPartition(),
new PruneFileScanPartition(),
new PushConjunctsIntoJdbcScan(),
new PushConjunctsIntoEsScan()

1
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
View File

@ -243,6 +243,7 @@ public enum RuleType {
COUNT_DISTINCT_REWRITE(RuleTypeClass.REWRITE),
INNER_TO_CROSS_JOIN(RuleTypeClass.REWRITE),
CROSS_TO_INNER_JOIN(RuleTypeClass.REWRITE),
PRUNE_EMPTY_PARTITION(RuleTypeClass.REWRITE),
// split limit
SPLIT_LIMIT(RuleTypeClass.REWRITE),

42
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java Normal file
View File

@ -0,0 +1,42 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import java.util.stream.Collectors;
/**
* Used to prune empty partition.
*/
public class PruneEmptyPartition extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalOlapScan().thenApply(ctx -> {
LogicalOlapScan scan = ctx.root;
OlapTable table = scan.getTable();
return scan.withSelectedPartitionIds(scan.getSelectedPartitionIds().stream()
.filter(partitionId -> table.getPartition(partitionId).hasData())
.collect(Collectors.toList()));
}).toRule(RuleType.PRUNE_EMPTY_PARTITION);
}
}

1
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
View File

@ -159,7 +159,6 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan
this.manuallySpecifiedPartitions = ImmutableList.copyOf(specifiedPartitions);
this.selectedPartitionIds = selectedPartitionIds.stream()
.filter(partitionId -> this.getTable().getPartition(partitionId) != null)
.filter(partitionId -> this.getTable().getPartition(partitionId).hasData())
.collect(Collectors.toList());
this.hints = Objects.requireNonNull(hints, "hints can not be null");
this.cacheSlotWithSlotName = Objects.requireNonNull(cacheSlotWithSlotName,

2
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java
View File

@ -113,8 +113,6 @@ class PruneOlapScanTabletTest implements MemoPatternMatchSupported {
result = "t1";
olapTable.getPartition(anyLong);
result = partition;
partition.hasData();
result = true;
partition.getIndex(anyLong);
result = index;
partition.getDistributionInfo();