database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[pipelineX](bug) Fix incorrect join operator judgement (#31690)

Browse Source 
* [pipelineX](bug) Fix incorrect join operator judgement

* update
This commit is contained in:
Gabriel
2024-03-03 19:21:11 +08:00
committed by yiguolei
parent 344e80fbbc
commit bc898e3cdc
7 changed files with 27 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
View File

@ -1205,9 +1205,6 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
hashJoinNode.setDistributeExprLists(distributeExprLists);
PlanFragment currentFragment = connectJoinNode(hashJoinNode, leftFragment, rightFragment, context, hashJoin);
if (joinType == JoinType.NULL_AWARE_LEFT_ANTI_JOIN) {
currentFragment.setHasNullAwareLeftAntiJoin(true);
}
if (JoinUtils.shouldColocateJoin(physicalHashJoin)) {
// TODO: add reason
hashJoinNode.setColocate(true, "");

2
fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
View File

@ -221,8 +221,6 @@ public class PlanTranslatorContext {
srcFragment.getBuilderRuntimeFilterIds().forEach(targetFragment::setBuilderRuntimeFilterIds);
targetFragment.setHasColocatePlanNode(targetFragment.hasColocatePlanNode()
|| srcFragment.hasColocatePlanNode());
targetFragment.setHasNullAwareLeftAntiJoin(targetFragment.isHasNullAwareLeftAntiJoin()
|| srcFragment.isHasNullAwareLeftAntiJoin());
this.planFragments.remove(srcFragment);
}

24
fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
View File

@ -784,10 +784,8 @@ public class HashJoinNode extends JoinNodeBase {
if (eqJoinConjuncts.isEmpty()) {
Preconditions.checkState(joinOp == JoinOperator.LEFT_SEMI_JOIN
|| joinOp == JoinOperator.LEFT_ANTI_JOIN);
if (joinOp == JoinOperator.LEFT_SEMI_JOIN) {
msg.hash_join_node.join_op = JoinOperator.NULL_AWARE_LEFT_SEMI_JOIN.toThrift();
} else if (joinOp == JoinOperator.LEFT_ANTI_JOIN) {
msg.hash_join_node.join_op = JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN.toThrift();
if (joinOp == JoinOperator.LEFT_SEMI_JOIN || joinOp == JoinOperator.LEFT_ANTI_JOIN) {
msg.hash_join_node.join_op = transformJoinOperator().toThrift();
}
// because eqJoinConjuncts mustn't be empty in thrift
// we have to use markJoinConjuncts instead
@ -970,4 +968,22 @@ public class HashJoinNode extends JoinNodeBase {
return slotRef;
}
}
private JoinOperator transformJoinOperator() {
boolean transformToNullAware = markJoinConjuncts != null && eqJoinConjuncts.isEmpty();
if (joinOp == JoinOperator.LEFT_ANTI_JOIN && transformToNullAware) {
return JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN;
} else if (joinOp == JoinOperator.LEFT_SEMI_JOIN && transformToNullAware) {
return JoinOperator.NULL_AWARE_LEFT_SEMI_JOIN;
}
return joinOp;
}
@Override
public boolean isNullAwareLeftAntiJoin() {
if (transformJoinOperator() == JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN) {
return true;
}
return children.stream().anyMatch(PlanNode::isNullAwareLeftAntiJoin);
}
}

1
fe/fe-core/src/main/java/org/apache/doris/planner/MultiCastPlanFragment.java
View File

@ -32,7 +32,6 @@ public class MultiCastPlanFragment extends PlanFragment {
super(planFragment.getFragmentId(), planFragment.getPlanRoot(), planFragment.getDataPartition(),
planFragment.getBuilderRuntimeFilterIds(), planFragment.getTargetRuntimeFilterIds());
this.hasColocatePlanNode = planFragment.hasColocatePlanNode;
this.hasNullAwareLeftAntiJoin = planFragment.hasNullAwareLeftAntiJoin;
this.outputPartition = DataPartition.RANDOM;
this.children.addAll(planFragment.getChildren());
}

10
fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java
View File

@ -149,8 +149,6 @@ public class PlanFragment extends TreeNode<PlanFragment> {
// has colocate plan node
protected boolean hasColocatePlanNode = false;
protected boolean hasNullAwareLeftAntiJoin = false;
private TResultSinkType resultSinkType = TResultSinkType.MYSQL_PROTOCAL;
/**
@ -473,11 +471,7 @@ public class PlanFragment extends TreeNode<PlanFragment> {
this.bucketNum = bucketNum;
}
public boolean isHasNullAwareLeftAntiJoin() {
return hasNullAwareLeftAntiJoin;
}
public void setHasNullAwareLeftAntiJoin(boolean hasNullAwareLeftAntiJoin) {
this.hasNullAwareLeftAntiJoin = hasNullAwareLeftAntiJoin;
public boolean hasNullAwareLeftAntiJoin() {
return planRoot.isNullAwareLeftAntiJoin();
}
}

4
fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
View File

@ -261,6 +261,10 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
this.fragment = fragment;
}
public boolean isNullAwareLeftAntiJoin() {
return children.stream().anyMatch(PlanNode::isNullAwareLeftAntiJoin);
}
public PlanFragment getFragment() {
return fragment;
}

2
fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
View File

@ -726,7 +726,7 @@ public abstract class ScanNode extends PlanNode {
return context != null
&& context.getSessionVariable().isIgnoreStorageDataDistribution()
&& context.getSessionVariable().getEnablePipelineXEngine()
&& !fragment.isHasNullAwareLeftAntiJoin()
&& !fragment.hasNullAwareLeftAntiJoin()
&& getScanRangeNum()
< ConnectContext.get().getSessionVariable().getParallelExecInstanceNum() * numBackends;
}