From 77b93ebc09fc9bd090b9175950595ef782d40ce8 Mon Sep 17 00:00:00 2001 From: jakevin Date: Fri, 9 Sep 2022 15:54:32 +0800 Subject: [PATCH] [enhancement](Nereids) add optionalAnd to simplify code (#12497) Add optionalAnd to avoid adding True which may make BE crash. Use optional to simplify code. --- .../glue/translator/ExpressionTranslator.java | 9 ++-- .../nereids/properties/DistributionSpec.java | 2 +- .../rules/exploration/join/JoinCommute.java | 3 +- .../exploration/join/JoinLAsscomHelper.java | 11 ++-- .../exploration/join/JoinReorderCommon.java | 15 ------ .../rewrite/logical/ApplyPullFilterOnAgg.java | 16 ++---- .../logical/FindHashConditionForJoin.java | 32 +++++------ .../logical/MergeConsecutiveFilters.java | 6 +-- .../logical/MergeConsecutiveLimits.java | 4 +- .../rules/rewrite/logical/MultiJoin.java | 40 ++++---------- .../rewrite/logical/PushApplyUnderFilter.java | 18 +++---- .../PushPredicateThroughAggregation.java | 19 +++---- .../logical/PushPredicateThroughJoin.java | 21 ++------ .../trees/plans/logical/LogicalJoin.java | 28 +++++----- .../plans/physical/AbstractPhysicalJoin.java | 13 +++-- .../doris/nereids/util/ExpressionUtils.java | 6 ++- .../apache/doris/nereids/util/PlanUtils.java | 53 +++++++++++++++++++ 17 files changed, 145 insertions(+), 151 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java index c18611aea5..017ec6b5b7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java @@ -153,16 +153,14 @@ public class ExpressionTranslator extends DefaultExpressionVisitor distribution = new PhysicalDistribute<>( this, child.getLogicalProperties(), new GroupPlan(child)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java index 129b655e5f..1fd5bbfaaa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.rules.exploration.OneExplorationRuleFactory; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; +import org.apache.doris.nereids.util.PlanUtils; import org.apache.doris.nereids.util.Utils; import java.util.ArrayList; @@ -63,7 +64,7 @@ public class JoinCommute extends OneExplorationRuleFactory { newJoin.getJoinReorderContext().setHasCommuteZigZag(true); } - return JoinReorderCommon.project(new ArrayList<>(join.getOutput()), newJoin).get(); + return PlanUtils.project(new ArrayList<>(join.getOutput()), newJoin).get(); }).toRule(RuleType.LOGICAL_JOIN_COMMUTATIVE); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinLAsscomHelper.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinLAsscomHelper.java index ac31083bde..860925d624 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinLAsscomHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinLAsscomHelper.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.PlanUtils; import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableSet; @@ -91,21 +92,21 @@ class JoinLAsscomHelper extends ThreeJoinHelper { newLeftProjectExpr.addAll(cOutput); } LogicalJoin newBottomJoin = new LogicalJoin<>(topJoin.getJoinType(), - newBottomHashJoinConjuncts, ExpressionUtils.andByOptional(newBottomNonHashJoinConjuncts), a, c, + newBottomHashJoinConjuncts, ExpressionUtils.optionalAnd(newBottomNonHashJoinConjuncts), a, c, bottomJoin.getJoinReorderContext()); newBottomJoin.getJoinReorderContext().setHasLAsscom(false); newBottomJoin.getJoinReorderContext().setHasCommute(false); - Plan left = JoinReorderCommon.project(newLeftProjectExpr, newBottomJoin).orElse(newBottomJoin); - Plan right = JoinReorderCommon.project(newRightProjectExprs, b).orElse(b); + Plan left = PlanUtils.projectOrSelf(newLeftProjectExpr, newBottomJoin); + Plan right = PlanUtils.projectOrSelf(newRightProjectExprs, b); LogicalJoin newTopJoin = new LogicalJoin<>(bottomJoin.getJoinType(), newTopHashJoinConjuncts, - ExpressionUtils.andByOptional(newTopNonHashJoinConjuncts), left, right, + ExpressionUtils.optionalAnd(newTopNonHashJoinConjuncts), left, right, topJoin.getJoinReorderContext()); newTopJoin.getJoinReorderContext().setHasLAsscom(true); - return JoinReorderCommon.project(new ArrayList<>(topJoin.getOutput()), newTopJoin).get(); + return PlanUtils.projectOrSelf(new ArrayList<>(topJoin.getOutput()), newTopJoin); } public static boolean check(Type type, LogicalJoin topJoin, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinReorderCommon.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinReorderCommon.java index 2a4f81138a..77eb09f014 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinReorderCommon.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinReorderCommon.java @@ -17,24 +17,9 @@ package org.apache.doris.nereids.rules.exploration.join; -import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; - -import java.util.List; -import java.util.Optional; - class JoinReorderCommon { public enum Type { INNER, OUTER } - - public static Optional project(List projectExprs, Plan plan) { - if (!projectExprs.isEmpty()) { - return Optional.of(new LogicalProject<>(projectExprs, plan)); - } else { - return Optional.empty(); - } - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnAgg.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnAgg.java index 3ddd25b719..e2be8d1524 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnAgg.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ApplyPullFilterOnAgg.java @@ -27,21 +27,21 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalApply; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.PlanUtils; import org.apache.doris.nereids.util.Utils; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.stream.Collectors; /** * Merge the correlated predicate and agg in the filter under apply. * And keep the unCorrelated predicate under agg. - * + *

* Use the correlated column as the group by column of agg, * the output column is the correlated column and the input column. - * + *

* before: * apply * / \ @@ -73,12 +73,6 @@ public class ApplyPullFilterOnAgg extends OneRewriteRuleFactory { return apply; } - LogicalFilter newUnCorrelatedFilter = null; - if (!unCorrelatedPredicate.isEmpty()) { - newUnCorrelatedFilter = new LogicalFilter<>(ExpressionUtils.and(unCorrelatedPredicate), - filter.child()); - } - List newAggOutput = new ArrayList<>(agg.getOutputExpressions()); List newGroupby = Utils.getCorrelatedSlots(correlatedPredicate, apply.getCorrelationSlot()); @@ -86,10 +80,10 @@ public class ApplyPullFilterOnAgg extends OneRewriteRuleFactory { newAggOutput.addAll(newGroupby.stream().map(NamedExpression.class::cast).collect(Collectors.toList())); LogicalAggregate newAgg = new LogicalAggregate<>( newGroupby, newAggOutput, - newUnCorrelatedFilter == null ? filter.child() : newUnCorrelatedFilter); + PlanUtils.filterOrSelf(unCorrelatedPredicate, filter.child())); return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), - Optional.ofNullable(ExpressionUtils.and(correlatedPredicate)), + ExpressionUtils.optionalAnd(correlatedPredicate), apply.left(), newAgg); }).toRule(RuleType.APPLY_PULL_FILTER_ON_AGG); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/FindHashConditionForJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/FindHashConditionForJoin.java index 50c2773b1b..b01c1930b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/FindHashConditionForJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/FindHashConditionForJoin.java @@ -34,14 +34,14 @@ import java.util.Optional; /** * this rule aims to find a conjunct list from on clause expression, which could * be used to build hash-table. - * + *

* For example: - * A join B on A.x=B.x and A.y>1 and A.x+1=B.x+B.y and A.z=B.z+A.x and (A.z=B.z or A.x=B.x) - * {A.x=B.x, A.x+1=B.x+B.y} could be used to build hash table, - * but {A.y>1, A.z=B.z+A.z, (A.z=B.z or A.x=B.x)} are not. - * + * A join B on A.x=B.x and A.y>1 and A.x+1=B.x+B.y and A.z=B.z+A.x and (A.z=B.z or A.x=B.x) + * {A.x=B.x, A.x+1=B.x+B.y} could be used to build hash table, + * but {A.y>1, A.z=B.z+A.z, (A.z=B.z or A.x=B.x)} are not. + *

* CAUTION: - * This rule must be applied after BindSlotReference + * This rule must be applied after BindSlotReference */ public class FindHashConditionForJoin extends OneRewriteRuleFactory { @Override @@ -49,21 +49,21 @@ public class FindHashConditionForJoin extends OneRewriteRuleFactory { return logicalJoin().then(join -> { Pair, List> pair = JoinUtils.extractExpressionForHashTable(join); List extractedHashJoinConjuncts = pair.first; - Optional remainedNonHashJoinConjuncts = Optional.of(ExpressionUtils.and(pair.second)); - if (!extractedHashJoinConjuncts.isEmpty()) { - List combinedHashJoinConjuncts = new ImmutableList.Builder() - .addAll(join.getHashJoinConjuncts()) - .addAll(extractedHashJoinConjuncts) - .build(); - return new LogicalJoin(join.getJoinType(), + Optional remainedNonHashJoinConjuncts = ExpressionUtils.optionalAnd(pair.second); + if (extractedHashJoinConjuncts.isEmpty()) { + return join; + } + + List combinedHashJoinConjuncts = new ImmutableList.Builder() + .addAll(join.getHashJoinConjuncts()) + .addAll(extractedHashJoinConjuncts) + .build(); + return new LogicalJoin<>(join.getJoinType(), combinedHashJoinConjuncts, remainedNonHashJoinConjuncts, Optional.empty(), Optional.empty(), join.left(), join.right()); - } else { - return join; - } }).toRule(RuleType.FIND_HASH_CONDITION_FOR_JOIN); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeConsecutiveFilters.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeConsecutiveFilters.java index 6ed9c3fd1d..dd7fca74f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeConsecutiveFilters.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeConsecutiveFilters.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.util.ExpressionUtils; @@ -46,12 +47,11 @@ public class MergeConsecutiveFilters extends OneRewriteRuleFactory { @Override public Rule build() { return logicalFilter(logicalFilter()).then(filter -> { - LogicalFilter childFilter = filter.child(); + LogicalFilter childFilter = filter.child(); Expression predicates = filter.getPredicates(); Expression childPredicates = childFilter.getPredicates(); Expression mergedPredicates = ExpressionUtils.and(predicates, childPredicates); - LogicalFilter mergedFilter = new LogicalFilter(mergedPredicates, childFilter.child()); - return mergedFilter; + return new LogicalFilter<>(mergedPredicates, childFilter.child()); }).toRule(RuleType.MERGE_CONSECUTIVE_FILTERS); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeConsecutiveLimits.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeConsecutiveLimits.java index 32d36318c8..6442dce0e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeConsecutiveLimits.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MergeConsecutiveLimits.java @@ -42,9 +42,9 @@ public class MergeConsecutiveLimits extends OneRewriteRuleFactory { @Override public Rule build() { return logicalLimit(logicalLimit()).then(upperLimit -> { - LogicalLimit bottomLimit = upperLimit.child(); + LogicalLimit bottomLimit = upperLimit.child(); List children = bottomLimit.children(); - return new LogicalLimit( + return new LogicalLimit<>( Math.min(upperLimit.getLimit(), bottomLimit.getLimit()), bottomLimit.getOffset(), children.get(0) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MultiJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MultiJoin.java index 7fda8114ed..1c343ba6fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MultiJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/MultiJoin.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.JoinUtils; +import org.apache.doris.nereids.util.PlanUtils; import java.util.ArrayList; import java.util.HashSet; @@ -56,18 +57,13 @@ public class MultiJoin extends PlanVisitor { /** * reorderJoinsAccordingToConditions + * * @return join or filter */ public Optional reorderJoinsAccordingToConditions() { if (joinInputs.size() >= 2) { Plan root = reorderJoinsAccordingToConditions(joinInputs, conjunctsForAllHashJoins); - if (!conjunctsKeepInFilter.isEmpty()) { - root = new LogicalFilter( - ExpressionUtils.and(conjunctsKeepInFilter), - root - ); - } - return Optional.of(root); + return Optional.of(PlanUtils.filterOrSelf(conjunctsKeepInFilter, root)); } return Optional.empty(); } @@ -94,20 +90,11 @@ public class MultiJoin extends PlanVisitor { conjuncts); List joinConditions = pair.first; conjunctsKeepInFilter = pair.second; - LogicalJoin join; - if (joinConditions.isEmpty()) { - join = new LogicalJoin(JoinType.CROSS_JOIN, - new ArrayList<>(), - Optional.empty(), - joinInputs.get(0), joinInputs.get(1)); - } else { - join = new LogicalJoin(JoinType.INNER_JOIN, - new ArrayList<>(), - Optional.of(ExpressionUtils.and(joinConditions)), - joinInputs.get(0), joinInputs.get(1)); - } - return join; + return new LogicalJoin<>(JoinType.INNER_JOIN, + new ArrayList<>(), + ExpressionUtils.optionalAnd(joinConditions), + joinInputs.get(0), joinInputs.get(1)); } // input size >= 3; Plan left = joinInputs.get(0); @@ -145,16 +132,9 @@ public class MultiJoin extends PlanVisitor { conjuncts); List joinConditions = pair.first; List nonJoinConditions = pair.second; - LogicalJoin join; - if (joinConditions.isEmpty()) { - join = new LogicalJoin(JoinType.CROSS_JOIN, new ArrayList(), - Optional.empty(), - left, right); - } else { - join = new LogicalJoin(JoinType.INNER_JOIN, new ArrayList<>(), - Optional.of(ExpressionUtils.and(joinConditions)), - left, right); - } + LogicalJoin join = new LogicalJoin<>(JoinType.INNER_JOIN, new ArrayList<>(), + ExpressionUtils.optionalAnd(joinConditions), + left, right); List newInputs = new ArrayList<>(); newInputs.add(join); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushApplyUnderFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushApplyUnderFilter.java index ed9b565090..fbc09b6e04 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushApplyUnderFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushApplyUnderFilter.java @@ -22,14 +22,15 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.GroupPlan; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalApply; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.PlanUtils; import org.apache.doris.nereids.util.Utils; import java.util.List; import java.util.Map; -import java.util.Optional; /** * Exchange apply and filter. @@ -63,17 +64,10 @@ public class PushApplyUnderFilter extends OneRewriteRuleFactory { return apply; } - if (unCorrelatedPredicate.isEmpty()) { - return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), - Optional.ofNullable(ExpressionUtils.and(correlatedPredicate)), - apply.left(), filter.child()); - } else { - LogicalFilter newFilter = new LogicalFilter<>( - ExpressionUtils.and(unCorrelatedPredicate), filter.child()); - return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), - Optional.ofNullable(ExpressionUtils.and(correlatedPredicate)), - apply.left(), newFilter); - } + Plan child = PlanUtils.filterOrSelf(unCorrelatedPredicate, filter.child()); + return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), + ExpressionUtils.optionalAnd(correlatedPredicate), + apply.left(), child); }).toRule(RuleType.PUSH_APPLY_UNDER_FILTER); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushPredicateThroughAggregation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushPredicateThroughAggregation.java index 4e8aa50398..9ed04483e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushPredicateThroughAggregation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushPredicateThroughAggregation.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.PlanUtils; import com.google.common.collect.Lists; @@ -89,19 +90,15 @@ public class PushPredicateThroughAggregation extends OneRewriteRuleFactory { } private Plan pushDownPredicate(LogicalFilter filter, LogicalAggregate aggregate, - List pushDownPredicates, List filterPredicates) { + List pushDownPredicates, List filterPredicates) { if (pushDownPredicates.size() == 0) { - //nothing pushed down, just return origin plan + // nothing pushed down, just return origin plan return filter; } - LogicalFilter bottomFilter = new LogicalFilter(ExpressionUtils.and(pushDownPredicates), - (Plan) aggregate.child(0)); - if (filterPredicates.isEmpty()) { - //all predicates are pushed down, just exchange filter and aggregate - return aggregate.withChildren(Lists.newArrayList(bottomFilter)); - } else { - aggregate = aggregate.withChildren(Lists.newArrayList(bottomFilter)); - return new LogicalFilter<>(ExpressionUtils.and(filterPredicates), aggregate); - } + LogicalFilter bottomFilter = new LogicalFilter<>(ExpressionUtils.and(pushDownPredicates), + aggregate.child(0)); + + aggregate = aggregate.withChildren(Lists.newArrayList(bottomFilter)); + return PlanUtils.filterOrSelf(filterPredicates, aggregate); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushPredicateThroughJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushPredicateThroughJoin.java index 6923e0509e..9859deb747 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushPredicateThroughJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushPredicateThroughJoin.java @@ -26,15 +26,14 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.PlanUtils; import com.google.common.collect.Lists; import java.util.List; import java.util.Objects; -import java.util.Optional; import java.util.Set; /** @@ -114,22 +113,12 @@ public class PushPredicateThroughJoin extends OneRewriteRuleFactory { private Plan pushDownPredicate(LogicalJoin joinPlan, List joinConditions, List leftPredicates, List rightPredicates) { - - Expression left = ExpressionUtils.and(leftPredicates); - Expression right = ExpressionUtils.and(rightPredicates); - //todo expr should optimize again using expr rewrite - Plan leftPlan = joinPlan.left(); - Plan rightPlan = joinPlan.right(); - if (!left.equals(BooleanLiteral.TRUE)) { - leftPlan = new LogicalFilter<>(left, leftPlan); - } - - if (!right.equals(BooleanLiteral.TRUE)) { - rightPlan = new LogicalFilter<>(right, rightPlan); - } + // todo expr should optimize again using expr rewrite + Plan leftPlan = PlanUtils.filterOrSelf(leftPredicates, joinPlan.left()); + Plan rightPlan = PlanUtils.filterOrSelf(rightPredicates, joinPlan.right()); return new LogicalJoin<>(joinPlan.getJoinType(), joinPlan.getHashJoinConjuncts(), - Optional.of(ExpressionUtils.and(joinConditions)), leftPlan, rightPlan); + ExpressionUtils.optionalAnd(joinConditions), leftPlan, rightPlan); } private Expression getJoinCondition(Expression predicate, Set leftOutputs, Set rightOutputs) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java index 61f34a84c3..6449ccadb0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java @@ -63,23 +63,24 @@ public class LogicalJoin hashJoinConjuncts, Optional condition, + public LogicalJoin(JoinType joinType, List hashJoinConjuncts, Optional otherJoinCondition, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) { this(joinType, hashJoinConjuncts, - condition, Optional.empty(), Optional.empty(), leftChild, rightChild); + otherJoinCondition, Optional.empty(), Optional.empty(), leftChild, rightChild); } - public LogicalJoin(JoinType joinType, List hashJoinConjuncts, Optional condition, + public LogicalJoin(JoinType joinType, List hashJoinConjuncts, Optional otherJoinCondition, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild, JoinReorderContext joinReorderContext) { - this(joinType, hashJoinConjuncts, condition, + this(joinType, hashJoinConjuncts, otherJoinCondition, Optional.empty(), Optional.empty(), leftChild, rightChild); this.joinReorderContext.copyFrom(joinReorderContext); } - public LogicalJoin(JoinType joinType, List hashJoinConjuncts, Optional condition, + public LogicalJoin(JoinType joinType, List hashJoinConjuncts, Optional otherJoinCondition, Optional groupExpression, Optional logicalProperties, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild, JoinReorderContext joinReorderContext) { - this(joinType, hashJoinConjuncts, condition, groupExpression, logicalProperties, leftChild, rightChild); + this(joinType, hashJoinConjuncts, otherJoinCondition, groupExpression, logicalProperties, leftChild, + rightChild); this.joinReorderContext.copyFrom(joinReorderContext); } @@ -118,15 +119,14 @@ public class LogicalJoin getOnClauseCondition() { - if (hashJoinConjuncts.isEmpty()) { - return otherJoinCondition; + Optional hashJoinCondition = ExpressionUtils.optionalAnd(hashJoinConjuncts); + + if (hashJoinCondition.isPresent() && otherJoinCondition.isPresent()) { + return ExpressionUtils.optionalAnd(hashJoinCondition.get(), otherJoinCondition.get()); } - Expression onClauseCondition = ExpressionUtils.and(hashJoinConjuncts); - if (otherJoinCondition.isPresent()) { - onClauseCondition = ExpressionUtils.and(onClauseCondition, otherJoinCondition.get()); - } - return Optional.of(onClauseCondition); + return hashJoinCondition.map(Optional::of) + .orElse(otherJoinCondition); } public JoinType getJoinType() { @@ -207,7 +207,7 @@ public class LogicalJoin R accept(PlanVisitor visitor, C context) { - return visitor.visitLogicalJoin((LogicalJoin) this, context); + return visitor.visitLogicalJoin(this, context); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java index b097cc5810..0a9efe9d68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java @@ -128,14 +128,13 @@ public abstract class AbstractPhysicalJoin< * @return the combination of hashJoinConjuncts and otherJoinCondition */ public Optional getOnClauseCondition() { - if (hashJoinConjuncts.isEmpty()) { - return otherJoinCondition; + Optional hashJoinCondition = ExpressionUtils.optionalAnd(hashJoinConjuncts); + + if (hashJoinCondition.isPresent() && otherJoinCondition.isPresent()) { + return ExpressionUtils.optionalAnd(hashJoinCondition.get(), otherJoinCondition.get()); } - Expression onClauseCondition = ExpressionUtils.and(hashJoinConjuncts); - if (otherJoinCondition.isPresent()) { - onClauseCondition = ExpressionUtils.and(onClauseCondition, otherJoinCondition.get()); - } - return Optional.of(onClauseCondition); + return hashJoinCondition.map(Optional::of) + .orElse(otherJoinCondition); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index 07ba450ea2..da5cd93bba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -79,7 +79,7 @@ public class ExpressionUtils { } } - public static Optional andByOptional(List expressions) { + public static Optional optionalAnd(List expressions) { if (expressions.isEmpty()) { return Optional.empty(); } else { @@ -87,6 +87,10 @@ public class ExpressionUtils { } } + public static Optional optionalAnd(Expression... expressions) { + return optionalAnd(Lists.newArrayList(expressions)); + } + public static Expression and(List expressions) { return combine(And.class, expressions); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java new file mode 100644 index 0000000000..96dc8fb99c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/PlanUtils.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.util; + +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; + +import java.util.List; +import java.util.Optional; + +/** + * Util for plan + */ +public class PlanUtils { + public static Optional> project(List projectExprs, Plan plan) { + if (projectExprs.isEmpty()) { + return Optional.empty(); + } + + return Optional.of(new LogicalProject<>(projectExprs, plan)); + } + + public static Plan projectOrSelf(List projectExprs, Plan plan) { + return project(projectExprs, plan).map(Plan.class::cast).orElse(plan); + } + + public static Optional> filter(List predicates, Plan plan) { + return ExpressionUtils.optionalAnd(predicates) + .map(predicate -> new LogicalFilter<>(predicate, plan)); + } + + public static Plan filterOrSelf(List predicates, Plan plan) { + return filter(predicates, plan).map(Plan.class::cast).orElse(plan); + } +}