From fd0cf78aa7e2276295fae0bdbf226eb8893d9b6e Mon Sep 17 00:00:00 2001 From: jakevin Date: Wed, 14 Sep 2022 10:32:05 +0800 Subject: [PATCH] [fix](Nereids): fix StatsCalculator compute project and correct commute join type. (#12539) --- .../org/apache/doris/nereids/rules/RuleSet.java | 2 ++ .../apache/doris/nereids/rules/RuleType.java | 10 +++++----- .../rules/exploration/join/JoinCommute.java | 4 ++-- .../logical/PushdownFilterThroughProject.java | 2 +- .../doris/nereids/stats/StatsCalculator.java | 17 +++++++++++++---- 5 files changed, 23 insertions(+), 12 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index 936834605b..70251110ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -34,6 +34,7 @@ import org.apache.doris.nereids.rules.implementation.LogicalSortToPhysicalQuickS import org.apache.doris.nereids.rules.implementation.LogicalTopNToPhysicalTopN; import org.apache.doris.nereids.rules.rewrite.AggregateDisassemble; import org.apache.doris.nereids.rules.rewrite.logical.MergeConsecutiveProjects; +import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughProject; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; @@ -48,6 +49,7 @@ public class RuleSet { .add(JoinCommute.OUTER_LEFT_DEEP) .add(JoinLAsscom.INNER) .add(JoinLAsscomProject.INNER) + .add(new PushdownFilterThroughProject()) .add(new MergeConsecutiveProjects()) .build(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 278bf7c929..98aac8e923 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -99,7 +99,6 @@ public enum RuleType { // Eliminate plan ELIMINATE_LIMIT(RuleTypeClass.REWRITE), ELIMINATE_FILTER(RuleTypeClass.REWRITE), - FIND_HASH_CONDITION_FOR_JOIN(RuleTypeClass.REWRITE), ROLLUP_AGG_SCAN(RuleTypeClass.REWRITE), ROLLUP_AGG_FILTER_SCAN(RuleTypeClass.REWRITE), @@ -107,8 +106,9 @@ public enum RuleType { ROLLUP_AGG_PROJECT_FILTER_SCAN(RuleTypeClass.REWRITE), ROLLUP_AGG_FILTER_PROJECT_SCAN(RuleTypeClass.REWRITE), OLAP_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE), - // Swap plan - SWAP_FILTER_AND_PROJECT(RuleTypeClass.REWRITE), + // Pushdown filter + PUSHDOWN_FILTER_THROUGH_PROJET(RuleTypeClass.REWRITE), + LOGICAL_LIMIT_TO_LOGICAL_EMPTY_RELATION_RULE(RuleTypeClass.REWRITE), SWAP_LIMIT_PROJECT(RuleTypeClass.REWRITE), REWRITE_SENTINEL(RuleTypeClass.REWRITE), @@ -152,8 +152,8 @@ public enum RuleType { return ruleTypeClass; } - public Rule build( - PatternMatcher patternMatcher) { + public + Rule build(PatternMatcher patternMatcher) { return patternMatcher.toRule(this); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java index 1fd5bbfaaa..1b40df3bc9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java @@ -50,11 +50,11 @@ public class JoinCommute extends OneExplorationRuleFactory { @Override public Rule build() { - return innerLogicalJoin() + return logicalJoin() .when(this::check) .then(join -> { LogicalJoin newJoin = new LogicalJoin<>( - join.getJoinType(), + join.getJoinType().swap(), join.getHashJoinConjuncts(), join.getOtherJoinCondition(), join.right(), join.left(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java index aab04a0ac6..c1e064a5c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughProject.java @@ -44,6 +44,6 @@ public class PushdownFilterThroughProject extends OneRewriteRuleFactory { project.child() ) ); - }).toRule(RuleType.SWAP_FILTER_AND_PROJECT); + }).toRule(RuleType.PUSHDOWN_FILTER_THROUGH_PROJET); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index ba25d62ce9..c75c939ede 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -70,7 +70,7 @@ import org.apache.doris.statistics.TableStats; import com.google.common.collect.Maps; -import java.util.AbstractMap; +import java.util.AbstractMap.SimpleEntry; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -341,14 +341,23 @@ public class StatsCalculator extends DefaultPlanVisitor StatsDeriveResult statsDeriveResult = groupExpression.getCopyOfChildStats(0); Map childColumnStats = statsDeriveResult.getSlotToColumnStats(); Map columnsStats = projections.stream().map(projection -> { + ColumnStats value = null; Set slots = projection.getInputSlots(); if (slots.isEmpty()) { - return new AbstractMap.SimpleEntry<>(projection.toSlot(), ColumnStats.createDefaultColumnStats()); + value = ColumnStats.createDefaultColumnStats(); } else { // TODO: just a trick here, need to do real project on column stats - return new AbstractMap.SimpleEntry<>(projection.toSlot(), - childColumnStats.get(slots.iterator().next())); + for (Slot slot : slots) { + if (childColumnStats.containsKey(slot)) { + value = childColumnStats.get(slot); + break; + } + } + if (value == null) { + value = ColumnStats.createDefaultColumnStats(); + } } + return new SimpleEntry<>(projection.toSlot(), value); }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (item1, item2) -> item1)); statsDeriveResult.setSlotToColumnStats(columnsStats); return statsDeriveResult;