database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](nereids) improve eliminate outerjoin in cascades (#24120)

Browse Source 
* eliminate outer join cascading
This commit is contained in:
minghong
2023-09-11 19:42:05 +08:00
committed by GitHub
parent 3d6d40db33
commit 115969c3fb
5 changed files with 559 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoin.java
View File

@ -21,6 +21,8 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
@ -31,7 +33,9 @@ import org.apache.doris.nereids.util.Utils;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSet.Builder;
import com.google.common.collect.Sets;
import java.util.Collection;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
@ -65,32 +69,50 @@ public class EliminateOuterJoin extends OneRewriteRuleFactory {
}
JoinType newJoinType = tryEliminateOuterJoin(join.getJoinType(), canFilterLeftNull, canFilterRightNull);
Set<Expression> conjuncts = new HashSet<>();
join.getHashJoinConjuncts().forEach(expression -> {
EqualTo equalTo = (EqualTo) expression;
if (canFilterLeftNull) {
JoinUtils.addIsNotNullIfNullableToCollection(equalTo.left(), conjuncts);
}
if (canFilterRightNull) {
JoinUtils.addIsNotNullIfNullableToCollection(equalTo.right(), conjuncts);
}
});
JoinUtils.JoinSlotCoverageChecker checker = new JoinUtils.JoinSlotCoverageChecker(
join.left().getOutput(),
join.right().getOutput());
join.getOtherJoinConjuncts().stream().filter(EqualTo.class::isInstance).forEach(expr -> {
EqualTo equalTo = (EqualTo) expr;
if (checker.isHashJoinCondition(equalTo)) {
if (canFilterLeftNull) {
JoinUtils.addIsNotNullIfNullableToCollection(equalTo.left(), conjuncts);
}
if (canFilterRightNull) {
JoinUtils.addIsNotNullIfNullableToCollection(equalTo.right(), conjuncts);
}
}
});
Set<Expression> conjuncts = Sets.newHashSet();
conjuncts.addAll(filter.getConjuncts());
return filter.withConjuncts(conjuncts).withChildren(join.withJoinType(newJoinType));
boolean conjunctsChanged = false;
if (!notNullSlots.isEmpty()) {
for (Slot slot : notNullSlots) {
Not isNotNull = new Not(new IsNull(slot));
isNotNull.isGeneratedIsNotNull = true;
conjunctsChanged |= conjuncts.add(isNotNull);
}
}
if (newJoinType.isInnerJoin()) {
/*
* for example: (A left join B on A.a=B.b) join C on B.x=C.x
* inner join condition B.x=C.x implies 'B.x is not null',
* by which the left outer join could be eliminated. Finally, the join transformed to
* (A join B on A.a=B.b) join C on B.x=C.x.
* This elimination can be processed recursively.
*
* TODO: is_not_null can also be inferred from A < B and so on
*/
conjunctsChanged |= join.getHashJoinConjuncts().stream()
.map(EqualTo.class::cast)
.map(equalTo ->
(EqualTo) JoinUtils.swapEqualToForChildrenOrder(equalTo, join.left().getOutputSet()))
.map(equalTo -> createIsNotNullIfNecessary(equalTo, conjuncts)
).anyMatch(Boolean::booleanValue);
JoinUtils.JoinSlotCoverageChecker checker = new JoinUtils.JoinSlotCoverageChecker(
join.left().getOutput(),
join.right().getOutput());
conjunctsChanged |= join.getOtherJoinConjuncts().stream().filter(EqualTo.class::isInstance)
.map(EqualTo.class::cast)
.filter(equalTo -> checker.isHashJoinCondition(equalTo))
.map(equalTo -> (EqualTo) JoinUtils.swapEqualToForChildrenOrder(equalTo,
join.left().getOutputSet()))
.map(equalTo ->
createIsNotNullIfNecessary(equalTo, conjuncts))
.anyMatch(Boolean::booleanValue);
}
if (conjunctsChanged) {
return filter.withConjuncts(conjuncts.stream().collect(ImmutableSet.toImmutableSet()))
.withChildren(join.withJoinType(newJoinType));
}
return filter.withChildren(join.withJoinType(newJoinType));
}).toRule(RuleType.ELIMINATE_OUTER_JOIN);
}
@ -112,4 +134,19 @@ public class EliminateOuterJoin extends OneRewriteRuleFactory {
}
return joinType;
}
private boolean createIsNotNullIfNecessary(EqualTo swapedEqualTo, Collection<Expression> container) {
boolean containerChanged = false;
if (swapedEqualTo.left().nullable()) {
Not not = new Not(new IsNull(swapedEqualTo.left()));
not.isGeneratedIsNotNull = true;
containerChanged |= container.add(not);
}
if (swapedEqualTo.right().nullable()) {
Not not = new Not(new IsNull(swapedEqualTo.right()));
not.isGeneratedIsNotNull = true;
containerChanged |= container.add(not);
}
return containerChanged;
}
}

25
fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java
View File

@ -27,7 +27,6 @@ import org.apache.doris.nereids.properties.DistributionSpecReplicated;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapContains;
@ -42,7 +41,6 @@ import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@ -63,23 +61,7 @@ public class JoinUtils {
}
/**
* for a given expr, if expr is nullable, add 'expr is not null' in to container.
* this is used to eliminate outer join.
* for example: (A left join B on A.a=B.b) join C on B.x=C.x
* inner join condition B.x=C.x implies that 'B.x is not null' can be used to filter B,
* with 'B.x is not null' predicate, we could eliminate outer join, and the join transformed to
* (A join B on A.a=B.b) join C on B.x=C.x
*/
public static void addIsNotNullIfNullableToCollection(Expression expr, Collection<Expression> container) {
if (expr.nullable()) {
Not not = new Not(new IsNull(expr));
not.isGeneratedIsNotNull = true;
container.add(not);
}
}
/**
* util class
* for a given equation, judge if it can be used as hash join condition
*/
public static final class JoinSlotCoverageChecker {
Set<ExprId> leftExprIds;
@ -90,6 +72,11 @@ public class JoinUtils {
rightExprIds = right.stream().map(Slot::getExprId).collect(Collectors.toSet());
}
JoinSlotCoverageChecker(Set<ExprId> left, Set<ExprId> right) {
leftExprIds = left;
rightExprIds = right;
}
/**
* PushDownExpressionInHashConjuncts ensure the "slots" is only one slot.
*/