diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferSetOperatorDistinct.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferSetOperatorDistinct.java
index 9bb24869e4..b9f99c0758 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferSetOperatorDistinct.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InferSetOperatorDistinct.java
@@ -50,7 +50,8 @@ public class InferSetOperatorDistinct extends OneRewriteRuleFactory {
                     }
 
                     List<Plan> newChildren = setOperation.children().stream()
-                            .map(child -> new LogicalAggregate<>(ImmutableList.copyOf(child.getOutput()), true, child))
+                            .map(child -> isAgg(child) ? child
+                                    : new LogicalAggregate<>(ImmutableList.copyOf(child.getOutput()), true, child))
                             .collect(ImmutableList.toImmutableList());
                     if (newChildren.equals(setOperation.children())) {
                         return null;
@@ -59,6 +60,11 @@ public class InferSetOperatorDistinct extends OneRewriteRuleFactory {
                 }).toRule(RuleType.INFER_SET_OPERATOR_DISTINCT);
     }
 
+    private boolean isAgg(Plan plan) {
+        return plan instanceof LogicalAggregate || (plan instanceof LogicalProject && plan.child(
+                0) instanceof LogicalAggregate);
+    }
+
     // if children exist NLJ, we can't infer distinct
     private boolean rejectNLJ(Plan plan) {
         if (plan instanceof LogicalProject) {