From 9cf9d568c722095090b722880a610c47156e2ada Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=A2=E5=81=A5?= Date: Mon, 18 Dec 2023 20:00:32 +0800 Subject: [PATCH] [feature](Nereids): add is null in predicate and put or expansion rule in rewriter (#28348) add is null in predicate put or expansion rule in rewrite job --- .../doris/nereids/jobs/executor/Rewriter.java | 3 +++ .../org/apache/doris/nereids/rules/RuleSet.java | 2 -- .../{exploration => rewrite}/OrExpansion.java | 15 ++++++++++----- .../doris/nereids/stats/FilterEstimation.java | 2 +- .../data/nereids_p0/union/or_expansion.out | 15 ++++++++++++++- .../suites/nereids_p0/union/or_expansion.groovy | 6 ++++++ 6 files changed, 34 insertions(+), 9 deletions(-) rename fe/fe-core/src/main/java/org/apache/doris/nereids/rules/{exploration => rewrite}/OrExpansion.java (95%) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 0ded776717..10da21d724 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -80,6 +80,7 @@ import org.apache.doris.nereids.rules.rewrite.MergeOneRowRelationIntoUnion; import org.apache.doris.nereids.rules.rewrite.MergeProjects; import org.apache.doris.nereids.rules.rewrite.MergeSetOperations; import org.apache.doris.nereids.rules.rewrite.NormalizeSort; +import org.apache.doris.nereids.rules.rewrite.OrExpansion; import org.apache.doris.nereids.rules.rewrite.PruneEmptyPartition; import org.apache.doris.nereids.rules.rewrite.PruneFileScanPartition; import org.apache.doris.nereids.rules.rewrite.PruneOlapScanPartition; @@ -405,6 +406,8 @@ public class Rewriter extends AbstractBatchJobExecutor { topic("rewrite cte sub-tree", custom(RuleType.REWRITE_CTE_CHILDREN, () -> new RewriteCteChildren(jobs)) ), + topic("or expansion", + topDown(new OrExpansion())), topic("whole plan check", custom(RuleType.ADJUST_NULLABLE, AdjustNullable::new) ) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index a937472c86..a38c33b76b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.rules; import org.apache.doris.nereids.rules.exploration.MergeProjectsCBO; -import org.apache.doris.nereids.rules.exploration.OrExpansion; import org.apache.doris.nereids.rules.exploration.TransposeAggSemiJoin; import org.apache.doris.nereids.rules.exploration.TransposeAggSemiJoinProject; import org.apache.doris.nereids.rules.exploration.join.InnerJoinLAsscom; @@ -122,7 +121,6 @@ public class RuleSet { .add(PushDownProjectThroughSemiJoin.INSTANCE) .add(TransposeAggSemiJoin.INSTANCE) .add(TransposeAggSemiJoinProject.INSTANCE) - .add(OrExpansion.INSTANCE) .build(); public static final List PUSH_DOWN_FILTERS = ImmutableList.of( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/OrExpansion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java similarity index 95% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/OrExpansion.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java index 73863ba1ae..199e8c8209 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/OrExpansion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java @@ -15,15 +15,16 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.rules.exploration; +package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.common.Pair; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.rewrite.PushDownExpressionsInHashCondition; +import org.apache.doris.nereids.rules.exploration.OneExplorationRuleFactory; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.Slot; @@ -72,7 +73,7 @@ public class OrExpansion extends OneExplorationRuleFactory { @Override public Rule build() { - return logicalJoin().when(JoinUtils::shouldNestedLoopJoin) + return logicalJoin(any(), any()).when(JoinUtils::shouldNestedLoopJoin) .when(join -> supportJoinType.contains(join.getJoinType()) && ConnectContext.get().getSessionVariable().getEnablePipelineEngine()) .thenApply(ctx -> { @@ -82,7 +83,7 @@ public class OrExpansion extends OneExplorationRuleFactory { //1. Try to split or conditions Pair, List> hashOtherConditions = splitOrCondition(join); - if (hashOtherConditions == null) { + if (hashOtherConditions == null || hashOtherConditions.first.size() <= 1) { return join; } @@ -221,7 +222,11 @@ public class OrExpansion extends OneExplorationRuleFactory { LogicalCTEProducer rightProducer) { List disjunctions = hashOtherConditions.first; List otherConditions = hashOtherConditions.second; - List notExprs = disjunctions.stream().map(Not::new).collect(Collectors.toList()); + // For null values, equalTo and not equalTo both return false + // To avoid it, we always return true when there is null + List notExprs = disjunctions.stream() + .map(e -> ExpressionUtils.or(new Not(e), new IsNull(e))) + .collect(ImmutableList.toImmutableList()); List joins = Lists.newArrayList(); for (int hashCondIdx = 0; hashCondIdx < disjunctions.size(); hashCondIdx++) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java index 0a7d6b70c3..9c04b59ce6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java @@ -367,7 +367,7 @@ public class FilterEstimation extends ExpressionVisitor