From 6fa239384d415c4181a4e9416b92608148ec95af Mon Sep 17 00:00:00 2001 From: jakevin Date: Wed, 22 Mar 2023 10:45:14 +0800 Subject: [PATCH] [refactor](Nereids) remove tabletPruned flag in LogicalOlapScan. (#17983) --- .../rewrite/logical/PruneOlapScanTablet.java | 18 +++++---- .../trees/plans/logical/LogicalOlapScan.java | 38 +++++++------------ 2 files changed, 24 insertions(+), 32 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneOlapScanTablet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneOlapScanTablet.java index b85218149c..a6c28caee5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneOlapScanTablet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneOlapScanTablet.java @@ -34,10 +34,11 @@ import org.apache.doris.planner.HashDistributionPruner; import org.apache.doris.planner.PartitionColumnFilter; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; +import com.google.common.collect.ImmutableList.Builder; import com.google.common.collect.Maps; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -51,18 +52,21 @@ public class PruneOlapScanTablet extends OneRewriteRuleFactory { @Override public Rule build() { return logicalFilter(logicalOlapScan()) - .when(filter -> !filter.child().isTabletPruned()) .then(filter -> { LogicalOlapScan olapScan = filter.child(); OlapTable table = olapScan.getTable(); - List selectedTabletIds = Lists.newArrayList(); + Builder selectedTabletIdsBuilder = ImmutableList.builder(); for (Long id : olapScan.getSelectedPartitionIds()) { Partition partition = table.getPartition(id); MaterializedIndex index = partition.getIndex(olapScan.getSelectedIndexId()); - selectedTabletIds.addAll(getSelectedTabletIds(filter.getConjuncts(), + selectedTabletIdsBuilder.addAll(getSelectedTabletIds(filter.getConjuncts(), index, partition.getDistributionInfo())); } - return filter.withChildren(olapScan.withSelectedTabletIds(ImmutableList.copyOf(selectedTabletIds))); + List selectedTabletIds = selectedTabletIdsBuilder.build(); + if (new HashSet(selectedTabletIds).equals(new HashSet(olapScan.getSelectedTabletIds()))) { + return null; + } + return filter.withChildren(olapScan.withSelectedTabletIds(selectedTabletIds)); }).toRule(RuleType.OLAP_SCAN_TABLET_PRUNE); } @@ -74,12 +78,12 @@ public class PruneOlapScanTablet extends OneRewriteRuleFactory { HashDistributionInfo hashInfo = (HashDistributionInfo) info; Map filterMap = Maps.newHashMap(); expressions.stream().map(ExpressionUtils::checkAndMaybeCommute).filter(Optional::isPresent) - .forEach(expr -> new ExpressionColumnFilterConverter(filterMap).convert(expr.get())); + .forEach(expr -> new ExpressionColumnFilterConverter(filterMap).convert(expr.get())); return new HashDistributionPruner(index.getTabletIdsInOrder(), hashInfo.getDistributionColumns(), filterMap, hashInfo.getBucketNum() - ).prune(); + ).prune(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 04c258bc6a..35126bb040 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -82,11 +82,6 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, */ private final List selectedTabletIds; - /** - * Status to indicate tablets are pruned or not. - */ - private final boolean tabletPruned; - /////////////////////////////////////////////////////////////////////////// // Members for partition ids. /////////////////////////////////////////////////////////////////////////// @@ -111,27 +106,27 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, public LogicalOlapScan(ObjectId id, OlapTable table, List qualifier) { this(id, table, qualifier, Optional.empty(), Optional.empty(), table.getPartitionIds(), false, - ImmutableList.of(), false, + ImmutableList.of(), -1, false, PreAggStatus.on(), ImmutableList.of(), ImmutableList.of()); } public LogicalOlapScan(ObjectId id, OlapTable table, List qualifier, List hints) { this(id, table, qualifier, Optional.empty(), Optional.empty(), table.getPartitionIds(), false, - ImmutableList.of(), false, + ImmutableList.of(), -1, false, PreAggStatus.on(), ImmutableList.of(), hints); } public LogicalOlapScan(ObjectId id, OlapTable table, List qualifier, List specifiedPartitions, List hints) { this(id, table, qualifier, Optional.empty(), Optional.empty(), - specifiedPartitions, false, ImmutableList.of(), false, + specifiedPartitions, false, ImmutableList.of(), -1, false, PreAggStatus.on(), specifiedPartitions, hints); } public LogicalOlapScan(ObjectId id, Table table, List qualifier) { this(id, table, qualifier, Optional.empty(), Optional.empty(), - ((OlapTable) table).getPartitionIds(), false, ImmutableList.of(), false, + ((OlapTable) table).getPartitionIds(), false, ImmutableList.of(), -1, false, PreAggStatus.on(), ImmutableList.of(), ImmutableList.of()); } @@ -141,15 +136,13 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, public LogicalOlapScan(ObjectId id, Table table, List qualifier, Optional groupExpression, Optional logicalProperties, List selectedPartitionIds, boolean partitionPruned, - List selectedTabletIds, boolean tabletPruned, - long selectedIndexId, boolean indexSelected, PreAggStatus preAggStatus, List partitions, - List hints) { + List selectedTabletIds, long selectedIndexId, boolean indexSelected, + PreAggStatus preAggStatus, List partitions, List hints) { super(id, PlanType.LOGICAL_OLAP_SCAN, table, qualifier, groupExpression, logicalProperties); this.selectedTabletIds = ImmutableList.copyOf(selectedTabletIds); this.partitionPruned = partitionPruned; - this.tabletPruned = tabletPruned; this.selectedIndexId = selectedIndexId <= 0 ? getTable().getBaseIndexId() : selectedIndexId; this.indexSelected = indexSelected; this.preAggStatus = preAggStatus; @@ -200,7 +193,6 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, && Objects.equals(selectedIndexId, ((LogicalOlapScan) o).selectedIndexId) && Objects.equals(indexSelected, ((LogicalOlapScan) o).indexSelected) && Objects.equals(selectedTabletIds, ((LogicalOlapScan) o).selectedTabletIds) - && Objects.equals(tabletPruned, ((LogicalOlapScan) o).tabletPruned) && Objects.equals(hints, ((LogicalOlapScan) o).hints); } @@ -209,45 +201,45 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, return Objects.hash(id, selectedPartitionIds, partitionPruned, selectedIndexId, indexSelected, - selectedTabletIds, tabletPruned, + selectedTabletIds, hints); } @Override public LogicalOlapScan withGroupExpression(Optional groupExpression) { return new LogicalOlapScan(id, (Table) table, qualifier, groupExpression, Optional.of(getLogicalProperties()), - selectedPartitionIds, partitionPruned, selectedTabletIds, tabletPruned, + selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints); } @Override public LogicalOlapScan withLogicalProperties(Optional logicalProperties) { return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), logicalProperties, - selectedPartitionIds, partitionPruned, selectedTabletIds, tabletPruned, + selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints); } public LogicalOlapScan withSelectedPartitionIds(List selectedPartitionIds) { return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), - selectedPartitionIds, true, selectedTabletIds, tabletPruned, + selectedPartitionIds, true, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints); } public LogicalOlapScan withMaterializedIndexSelected(PreAggStatus preAgg, long indexId) { return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), - selectedPartitionIds, partitionPruned, selectedTabletIds, tabletPruned, + selectedPartitionIds, partitionPruned, selectedTabletIds, indexId, true, preAgg, manuallySpecifiedPartitions, hints); } public LogicalOlapScan withSelectedTabletIds(List selectedTabletIds) { return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), - selectedPartitionIds, partitionPruned, selectedTabletIds, true, + selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints); } public LogicalOlapScan withPreAggStatus(PreAggStatus preAggStatus) { return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), - selectedPartitionIds, partitionPruned, selectedTabletIds, true, + selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints); } @@ -260,10 +252,6 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, return partitionPruned; } - public boolean isTabletPruned() { - return tabletPruned; - } - public List getSelectedTabletIds() { return selectedTabletIds; }