diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 0ded776717..10da21d724 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -80,6 +80,7 @@ import org.apache.doris.nereids.rules.rewrite.MergeOneRowRelationIntoUnion; import org.apache.doris.nereids.rules.rewrite.MergeProjects; import org.apache.doris.nereids.rules.rewrite.MergeSetOperations; import org.apache.doris.nereids.rules.rewrite.NormalizeSort; +import org.apache.doris.nereids.rules.rewrite.OrExpansion; import org.apache.doris.nereids.rules.rewrite.PruneEmptyPartition; import org.apache.doris.nereids.rules.rewrite.PruneFileScanPartition; import org.apache.doris.nereids.rules.rewrite.PruneOlapScanPartition; @@ -405,6 +406,8 @@ public class Rewriter extends AbstractBatchJobExecutor { topic("rewrite cte sub-tree", custom(RuleType.REWRITE_CTE_CHILDREN, () -> new RewriteCteChildren(jobs)) ), + topic("or expansion", + topDown(new OrExpansion())), topic("whole plan check", custom(RuleType.ADJUST_NULLABLE, AdjustNullable::new) ) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index a937472c86..a38c33b76b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.rules; import org.apache.doris.nereids.rules.exploration.MergeProjectsCBO; -import org.apache.doris.nereids.rules.exploration.OrExpansion; import org.apache.doris.nereids.rules.exploration.TransposeAggSemiJoin; import org.apache.doris.nereids.rules.exploration.TransposeAggSemiJoinProject; import org.apache.doris.nereids.rules.exploration.join.InnerJoinLAsscom; @@ -122,7 +121,6 @@ public class RuleSet { .add(PushDownProjectThroughSemiJoin.INSTANCE) .add(TransposeAggSemiJoin.INSTANCE) .add(TransposeAggSemiJoinProject.INSTANCE) - .add(OrExpansion.INSTANCE) .build(); public static final List PUSH_DOWN_FILTERS = ImmutableList.of( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/OrExpansion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java similarity index 95% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/OrExpansion.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java index 73863ba1ae..199e8c8209 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/OrExpansion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OrExpansion.java @@ -15,15 +15,16 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.rules.exploration; +package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.common.Pair; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.rewrite.PushDownExpressionsInHashCondition; +import org.apache.doris.nereids.rules.exploration.OneExplorationRuleFactory; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.Slot; @@ -72,7 +73,7 @@ public class OrExpansion extends OneExplorationRuleFactory { @Override public Rule build() { - return logicalJoin().when(JoinUtils::shouldNestedLoopJoin) + return logicalJoin(any(), any()).when(JoinUtils::shouldNestedLoopJoin) .when(join -> supportJoinType.contains(join.getJoinType()) && ConnectContext.get().getSessionVariable().getEnablePipelineEngine()) .thenApply(ctx -> { @@ -82,7 +83,7 @@ public class OrExpansion extends OneExplorationRuleFactory { //1. Try to split or conditions Pair, List> hashOtherConditions = splitOrCondition(join); - if (hashOtherConditions == null) { + if (hashOtherConditions == null || hashOtherConditions.first.size() <= 1) { return join; } @@ -221,7 +222,11 @@ public class OrExpansion extends OneExplorationRuleFactory { LogicalCTEProducer rightProducer) { List disjunctions = hashOtherConditions.first; List otherConditions = hashOtherConditions.second; - List notExprs = disjunctions.stream().map(Not::new).collect(Collectors.toList()); + // For null values, equalTo and not equalTo both return false + // To avoid it, we always return true when there is null + List notExprs = disjunctions.stream() + .map(e -> ExpressionUtils.or(new Not(e), new IsNull(e))) + .collect(ImmutableList.toImmutableList()); List joins = Lists.newArrayList(); for (int hashCondIdx = 0; hashCondIdx < disjunctions.size(); hashCondIdx++) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java index 0a7d6b70c3..9c04b59ce6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java @@ -367,7 +367,7 @@ public class FilterEstimation extends ExpressionVisitor