diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 8210c1a082..6aff68e6b4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -1224,7 +1224,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
             setPlanRoot(leftFragment, nestedLoopJoinNode, nestedLoopJoin);
             // TODO: what's this? do we really need to set this?
             rightFragment.getPlanRoot().setCompactData(false);
-            context.removePlanFragment(rightFragment);
+            context.mergePlanFragment(rightFragment, leftFragment);
             for (PlanFragment rightChild : rightFragment.getChildren()) {
                 leftFragment.addChild(rightChild);
             }
@@ -1557,7 +1557,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
             int childrenSize = childrenFragments.size();
             setOperationFragment = childrenFragments.get(childrenSize - 1);
             for (int i = childrenSize - 2; i >= 0; i--) {
-                context.removePlanFragment(childrenFragments.get(i));
+                context.mergePlanFragment(childrenFragments.get(i), setOperationFragment);
                 for (PlanFragment child : childrenFragments.get(i).getChildren()) {
                     setOperationFragment.addChild(child);
                 }
@@ -1995,9 +1995,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
         hashJoinNode.setChild(0, leftFragment.getPlanRoot());
         hashJoinNode.setChild(1, rightFragment.getPlanRoot());
         setPlanRoot(leftFragment, hashJoinNode, join);
-        rightFragment.getTargetRuntimeFilterIds().forEach(leftFragment::setTargetRuntimeFilterIds);
-        rightFragment.getBuilderRuntimeFilterIds().forEach(leftFragment::setBuilderRuntimeFilterIds);
-        context.removePlanFragment(rightFragment);
+        context.mergePlanFragment(rightFragment, leftFragment);
         for (PlanFragment rightChild : rightFragment.getChildren()) {
             leftFragment.addChild(rightChild);
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
index 5a474b5492..24d6d6e1f7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java
@@ -137,8 +137,10 @@ public class PlanTranslatorContext {
         slotIdToExprId.put(slotRef.getDesc().getId(), exprId);
     }
 
-    public void removePlanFragment(PlanFragment planFragment) {
-        this.planFragments.remove(planFragment);
+    public void mergePlanFragment(PlanFragment srcFragment, PlanFragment targetFragment) {
+        srcFragment.getTargetRuntimeFilterIds().forEach(targetFragment::setTargetRuntimeFilterIds);
+        srcFragment.getBuilderRuntimeFilterIds().forEach(targetFragment::setBuilderRuntimeFilterIds);
+        this.planFragments.remove(srcFragment);
     }
 
     public SlotRef findSlotRef(ExprId exprId) {