From 20fa1eff65c8e3578da4a123f71812097b09eb75 Mon Sep 17 00:00:00 2001 From: AKIRA <33112463+Kikyou1997@users.noreply.github.com> Date: Wed, 25 Oct 2023 17:37:10 +0800 Subject: [PATCH] [enhancement](Nereids) Filter no data partition after partition prune (#25456) --- .../doris/nereids/jobs/executor/Rewriter.java | 2 + .../apache/doris/nereids/rules/RuleType.java | 1 + .../rules/rewrite/PruneEmptyPartition.java | 42 +++++++++++++++++++ .../trees/plans/logical/LogicalOlapScan.java | 1 - .../rewrite/PruneOlapScanTabletTest.java | 2 - 5 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 5003e5e1ea..9e450c322c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -80,6 +80,7 @@ import org.apache.doris.nereids.rules.rewrite.MergeOneRowRelationIntoUnion; import org.apache.doris.nereids.rules.rewrite.MergeProjects; import org.apache.doris.nereids.rules.rewrite.MergeSetOperations; import org.apache.doris.nereids.rules.rewrite.NormalizeSort; +import org.apache.doris.nereids.rules.rewrite.PruneEmptyPartition; import org.apache.doris.nereids.rules.rewrite.PruneFileScanPartition; import org.apache.doris.nereids.rules.rewrite.PruneOlapScanPartition; import org.apache.doris.nereids.rules.rewrite.PruneOlapScanTablet; @@ -300,6 +301,7 @@ public class Rewriter extends AbstractBatchJobExecutor { topic("Table/Physical optimization", topDown( new PruneOlapScanPartition(), + new PruneEmptyPartition(), new PruneFileScanPartition(), new PushConjunctsIntoJdbcScan(), new PushConjunctsIntoEsScan() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index b8078cdae9..505c7db91a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -243,6 +243,7 @@ public enum RuleType { COUNT_DISTINCT_REWRITE(RuleTypeClass.REWRITE), INNER_TO_CROSS_JOIN(RuleTypeClass.REWRITE), CROSS_TO_INNER_JOIN(RuleTypeClass.REWRITE), + PRUNE_EMPTY_PARTITION(RuleTypeClass.REWRITE), // split limit SPLIT_LIMIT(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java new file mode 100644 index 0000000000..5b3154e29f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; + +import java.util.stream.Collectors; + +/** + * Used to prune empty partition. + */ +public class PruneEmptyPartition extends OneRewriteRuleFactory { + + @Override + public Rule build() { + return logicalOlapScan().thenApply(ctx -> { + LogicalOlapScan scan = ctx.root; + OlapTable table = scan.getTable(); + return scan.withSelectedPartitionIds(scan.getSelectedPartitionIds().stream() + .filter(partitionId -> table.getPartition(partitionId).hasData()) + .collect(Collectors.toList())); + }).toRule(RuleType.PRUNE_EMPTY_PARTITION); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 03c18d8f2f..6916631898 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -159,7 +159,6 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan this.manuallySpecifiedPartitions = ImmutableList.copyOf(specifiedPartitions); this.selectedPartitionIds = selectedPartitionIds.stream() .filter(partitionId -> this.getTable().getPartition(partitionId) != null) - .filter(partitionId -> this.getTable().getPartition(partitionId).hasData()) .collect(Collectors.toList()); this.hints = Objects.requireNonNull(hints, "hints can not be null"); this.cacheSlotWithSlotName = Objects.requireNonNull(cacheSlotWithSlotName, diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java index 38cdc115de..e7940cdfb9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTabletTest.java @@ -113,8 +113,6 @@ class PruneOlapScanTabletTest implements MemoPatternMatchSupported { result = "t1"; olapTable.getPartition(anyLong); result = partition; - partition.hasData(); - result = true; partition.getIndex(anyLong); result = index; partition.getDistributionInfo();