database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[opt](Nereids) enhance properties regulator checking (#34603)

Browse Source 
Enhance properties regulator checking:
(1) right bucket shuffle restriction takes effective only when either side has NATUAL shuffle type.
(2) enhance bothSideShuffleKeysAreSameOrder checking if taking EquivalenceExprIds into consideration.


Co-authored-by: zhongjian.xzj <zhongjian.xzj@zhongjianxzjdeMacBook-Pro.local>
This commit is contained in:
xzj7019
2024-05-11 10:46:30 +08:00
committed by yiguolei
parent d6f300ec62
commit db15c811f8
8 changed files with 244 additions and 230 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java
View File

@ -216,10 +216,14 @@ public class ChildrenPropertiesRegulator extends PlanVisitor<Boolean, Void> {
}
}
private boolean couldNotRightBucketShuffleJoin(JoinType joinType) {
return joinType == JoinType.RIGHT_ANTI_JOIN
private boolean couldNotRightBucketShuffleJoin(JoinType joinType, DistributionSpecHash leftHashSpec,
DistributionSpecHash rightHashSpec) {
boolean isJoinTypeInScope = (joinType == JoinType.RIGHT_ANTI_JOIN
|| joinType == JoinType.RIGHT_OUTER_JOIN
|| joinType == JoinType.FULL_OUTER_JOIN;
|| joinType == JoinType.FULL_OUTER_JOIN);
boolean isSpecInScope = (leftHashSpec.getShuffleType() == ShuffleType.NATURAL
|| rightHashSpec.getShuffleType() == ShuffleType.NATURAL);
return isJoinTypeInScope && isSpecInScope;
}
@Override
@ -254,7 +258,7 @@ public class ChildrenPropertiesRegulator extends PlanVisitor<Boolean, Void> {
if (JoinUtils.couldColocateJoin(leftHashSpec, rightHashSpec)) {
// check colocate join with scan
return true;
} else if (couldNotRightBucketShuffleJoin(hashJoin.getJoinType())) {
} else if (couldNotRightBucketShuffleJoin(hashJoin.getJoinType(), leftHashSpec, rightHashSpec)) {
// right anti, right outer, full outer join could not do bucket shuffle join
// TODO remove this after we refactor coordinator
updatedForLeft = Optional.of(calAnotherSideRequired(
@ -482,8 +486,32 @@ public class ChildrenPropertiesRegulator extends PlanVisitor<Boolean, Void> {
private boolean bothSideShuffleKeysAreSameOrder(
DistributionSpecHash notShuffleSideOutput, DistributionSpecHash shuffleSideOutput,
DistributionSpecHash notShuffleSideRequired, DistributionSpecHash shuffleSideRequired) {
return shuffleSideOutput.getOrderedShuffledColumns().equals(
calAnotherSideRequiredShuffleIds(notShuffleSideOutput, notShuffleSideRequired, shuffleSideRequired));
List<ExprId> shuffleSideOutputList = shuffleSideOutput.getOrderedShuffledColumns();
List<ExprId> notShuffleSideOutputList = calAnotherSideRequiredShuffleIds(notShuffleSideOutput,
notShuffleSideRequired, shuffleSideRequired);
if (shuffleSideOutputList.size() != notShuffleSideOutputList.size()) {
return false;
} else if (shuffleSideOutputList.equals(notShuffleSideOutputList)) {
return true;
} else {
boolean isSatisfy = true;
for (int i = 0; i < shuffleSideOutputList.size() && isSatisfy; i++) {
ExprId shuffleSideExprId = shuffleSideOutputList.get(i);
boolean found = false;
for (int j = 0; j < notShuffleSideOutputList.size() && !found; j++) {
ExprId notShuffleSideExprId = notShuffleSideOutputList.get(j);
if (shuffleSideExprId.equals(notShuffleSideExprId)
|| shuffleSideOutput.getEquivalenceExprIdsOf(shuffleSideExprId)
.contains(notShuffleSideExprId)) {
found = true;
}
}
if (!found) {
isSatisfy = false;
}
}
return isSatisfy;
}
}
/**