database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Improve](Nereids): refactor eliminate outer join (#13402)

Browse Source 
Refactor eliminate outer join #12985

Evaluate the expression with ConstantFoldRule. If the evaluation result is NULL or FALSE, then the elimination condition is satisfied.
This commit is contained in:
shee
2022-11-02 17:39:05 +08:00
committed by GitHub
parent 1bafb26217
commit a871fef815
7 changed files with 129 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
View File

@ -39,7 +39,7 @@ import org.apache.doris.nereids.rules.implementation.LogicalProjectToPhysicalPro
import org.apache.doris.nereids.rules.implementation.LogicalSortToPhysicalQuickSort;
import org.apache.doris.nereids.rules.implementation.LogicalTopNToPhysicalTopN;
import org.apache.doris.nereids.rules.rewrite.AggregateDisassemble;
import org.apache.doris.nereids.rules.rewrite.logical.EliminateOuter;
import org.apache.doris.nereids.rules.rewrite.logical.EliminateOuterJoin;
import org.apache.doris.nereids.rules.rewrite.logical.MergeFilters;
import org.apache.doris.nereids.rules.rewrite.logical.MergeLimits;
import org.apache.doris.nereids.rules.rewrite.logical.MergeProjects;
@ -79,7 +79,7 @@ public class RuleSet {
new PushdownExpressionsInHashCondition(),
new PushdownProjectThroughLimit(),
new PushdownFilterThroughProject(),
EliminateOuter.INSTANCE,
new EliminateOuterJoin(),
new MergeProjects(),
new MergeFilters(),
new MergeLimits());

2
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
View File

@ -107,7 +107,7 @@ public enum RuleType {
// Eliminate plan
ELIMINATE_LIMIT(RuleTypeClass.REWRITE),
ELIMINATE_FILTER(RuleTypeClass.REWRITE),
ELIMINATE_OUTER(RuleTypeClass.REWRITE),
ELIMINATE_OUTER_JOIN(RuleTypeClass.REWRITE),
FIND_HASH_CONDITION_FOR_JOIN(RuleTypeClass.REWRITE),
MATERIALIZED_INDEX_AGG_SCAN(RuleTypeClass.REWRITE),
MATERIALIZED_INDEX_AGG_FILTER_SCAN(RuleTypeClass.REWRITE),

91
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuter.java
View File

@ -1,91 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite.logical;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.ImmutableMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Eliminate outer.
*/
public class EliminateOuter extends OneRewriteRuleFactory {
public static EliminateOuter INSTANCE = new EliminateOuter();
// right nullable
public static Map<JoinType, JoinType> ELIMINATE_LEFT_MAP = ImmutableMap.of(
JoinType.LEFT_OUTER_JOIN, JoinType.INNER_JOIN,
JoinType.FULL_OUTER_JOIN, JoinType.RIGHT_OUTER_JOIN
);
// left nullable
public static Map<JoinType, JoinType> ELIMINATE_RIGHT_MAP = ImmutableMap.of(
JoinType.RIGHT_OUTER_JOIN, JoinType.INNER_JOIN,
JoinType.FULL_OUTER_JOIN, JoinType.LEFT_OUTER_JOIN
);
@Override
public Rule build() {
return logicalFilter(logicalJoin())
.when(filter -> filter.child().getJoinType().isOuterJoin())
.then(filter -> {
List<Expression> predicates = ExpressionUtils.extractConjunction(filter.getPredicates());
Set<Slot> notNullSlots = new HashSet<>();
for (Expression predicate : predicates) {
// TODO: more case.
if (predicate instanceof ComparisonPredicate) {
notNullSlots.addAll(predicate.getInputSlots());
}
}
LogicalJoin<GroupPlan, GroupPlan> join = filter.child();
JoinType joinType = join.getJoinType();
if (!joinType.isLeftOuterJoin() && ExpressionUtils.isIntersecting(join.left().getOutputSet(),
notNullSlots)) {
joinType = ELIMINATE_RIGHT_MAP.get(joinType);
}
if (!joinType.isRightOuterJoin() && ExpressionUtils.isIntersecting(join.right().getOutputSet(),
notNullSlots)) {
joinType = ELIMINATE_LEFT_MAP.get(joinType);
}
if (joinType == join.getJoinType()) {
return null;
}
return new LogicalFilter<>(filter.getPredicates(),
new LogicalJoin<>(joinType,
join.getHashJoinConjuncts(), join.getOtherJoinConjuncts(),
join.left(), join.right()));
}).toRule(RuleType.ELIMINATE_OUTER);
}
}

98
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoin.java Normal file
View File

@ -0,0 +1,98 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.rules.rewrite.logical;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewriteContext;
import org.apache.doris.nereids.rules.expression.rewrite.rules.FoldConstantRule;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.collect.Maps;
import java.util.List;
import java.util.Map;
/**
* Eliminate outer join.
*/
public class EliminateOuterJoin extends OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalFilter(
logicalJoin().when(join -> join.getJoinType().isOuterJoin())
).then(filter -> {
LogicalJoin<GroupPlan, GroupPlan> join = filter.child();
List<Expression> conjuncts = filter.getConjuncts();
List<Expression> leftPredicates = ExpressionUtils.extractCoveredConjunction(conjuncts,
join.left().getOutputSet());
List<Expression> rightPredicates = ExpressionUtils.extractCoveredConjunction(conjuncts,
join.right().getOutputSet());
boolean canFilterLeftNull = canFilterNull(leftPredicates);
boolean canFilterRightNull = canFilterNull(rightPredicates);
JoinType newJoinType = tryEliminateOuterJoin(join.getJoinType(), canFilterLeftNull,
canFilterRightNull);
if (newJoinType == join.getJoinType()) {
return filter;
} else {
return filter.withChildren(join.withJoinType(newJoinType));
}
}).toRule(RuleType.ELIMINATE_OUTER_JOIN);
}
private JoinType tryEliminateOuterJoin(JoinType joinType, boolean canFilterLeftNull, boolean canFilterRightNull) {
if (joinType.isRightOuterJoin() && canFilterLeftNull) {
return JoinType.INNER_JOIN;
}
if (joinType.isLeftOuterJoin() && canFilterRightNull) {
return JoinType.INNER_JOIN;
}
if (joinType.isFullOuterJoin() && canFilterLeftNull && canFilterRightNull) {
return JoinType.INNER_JOIN;
}
if (joinType.isFullOuterJoin() && canFilterLeftNull) {
return JoinType.LEFT_OUTER_JOIN;
}
if (joinType.isFullOuterJoin() && canFilterRightNull) {
return JoinType.RIGHT_OUTER_JOIN;
}
return joinType;
}
private boolean canFilterNull(List<Expression> predicates) {
Literal nullLiteral = Literal.of(null);
for (Expression predicate : predicates) {
Map<Expression, Expression> replaceMap = Maps.newHashMap();
predicate.getInputSlots().forEach(slot -> replaceMap.put(slot, nullLiteral));
Expression evalExpr = FoldConstantRule.INSTANCE.rewrite(ExpressionUtils.replace(predicate, replaceMap),
new ExpressionRewriteContext(null));
if (nullLiteral.equals(evalExpr) || BooleanLiteral.FALSE.equals(evalExpr)) {
return true;
}
}
return false;
}
}

4
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
View File

@ -254,4 +254,8 @@ public class LogicalJoin<LEFT_CHILD_TYPE extends Plan, RIGHT_CHILD_TYPE extends
return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, children.get(0), children.get(1),
joinReorderContext);
}
public LogicalJoin withJoinType(JoinType joinType) {
return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, left(), right(), joinReorderContext);
}
}

14
fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
View File

@ -300,4 +300,18 @@ public class ExpressionUtils {
public static boolean isAllNullLiteral(List<Expression> children) {
return children.stream().allMatch(c -> c instanceof NullLiteral);
}
/**
* extract the predicate that is covered by `slots`
*/
public static List<Expression> extractCoveredConjunction(List<Expression> predicates, Set<Slot> slots) {
List<Expression> coveredPredicates = Lists.newArrayList();
for (Expression predicate : predicates) {
if (slots.containsAll(predicate.getInputSlots())) {
coveredPredicates.add(predicate);
}
}
return coveredPredicates;
}
}

20
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterTest.java
View File

@ -39,12 +39,12 @@ class EliminateOuterTest implements PatternMatchSupported {
@Test
void testEliminateLeft() {
LogicalPlan plan = new LogicalPlanBuilder(scan1)
.hashJoinUsing(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(1, 1)) // t1.id = t2.id
.filter(new GreaterThan(scan2.getOutput().get(1), Literal.of(1)))
.hashJoinUsing(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0)) // t1.id = t2.id
.filter(new GreaterThan(scan2.getOutput().get(0), Literal.of(1)))
.build();
PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
.applyTopDown(EliminateOuter.INSTANCE)
.applyTopDown(new EliminateOuterJoin())
.matchesFromRoot(
logicalFilter(
logicalJoin().when(join -> join.getJoinType().isInnerJoin())
@ -55,12 +55,12 @@ class EliminateOuterTest implements PatternMatchSupported {
@Test
void testEliminateRight() {
LogicalPlan plan = new LogicalPlanBuilder(scan1)
.hashJoinUsing(scan2, JoinType.RIGHT_OUTER_JOIN, Pair.of(1, 1)) // t1.id = t2.id
.filter(new GreaterThan(scan1.getOutput().get(1), Literal.of(1)))
.hashJoinUsing(scan2, JoinType.RIGHT_OUTER_JOIN, Pair.of(0, 0)) // t1.id = t2.id
.filter(new GreaterThan(scan1.getOutput().get(0), Literal.of(1)))
.build();
PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
.applyTopDown(EliminateOuter.INSTANCE)
.applyTopDown(new EliminateOuterJoin())
.matchesFromRoot(
logicalFilter(
logicalJoin().when(join -> join.getJoinType().isInnerJoin())
@ -71,14 +71,14 @@ class EliminateOuterTest implements PatternMatchSupported {
@Test
void testEliminateBoth() {
LogicalPlan plan = new LogicalPlanBuilder(scan1)
.hashJoinUsing(scan2, JoinType.FULL_OUTER_JOIN, Pair.of(1, 1)) // t1.id = t2.id
.hashJoinUsing(scan2, JoinType.FULL_OUTER_JOIN, Pair.of(0, 0)) // t1.id = t2.id
.filter(new And(
new GreaterThan(scan2.getOutput().get(1), Literal.of(1)),
new GreaterThan(scan1.getOutput().get(1), Literal.of(1))))
new GreaterThan(scan2.getOutput().get(0), Literal.of(1)),
new GreaterThan(scan1.getOutput().get(0), Literal.of(1))))
.build();
PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
.applyTopDown(EliminateOuter.INSTANCE)
.applyTopDown(new EliminateOuterJoin())
.matchesFromRoot(
logicalFilter(
logicalJoin().when(join -> join.getJoinType().isInnerJoin())