From 17ca75f83400fdf66153de7946b39946de1cacc3 Mon Sep 17 00:00:00 2001 From: jakevin Date: Fri, 24 Nov 2023 10:06:04 +0800 Subject: [PATCH] [chore](Nereids): add eager aggregate into rules (#27505) Add `Eager Aggregate` rules into Rewrite rules. --- .../doris/nereids/jobs/executor/Rewriter.java | 18 +++++----- .../rewrite/PushDownCountThroughJoinTest.java | 36 ++++--------------- .../PushDownDistinctThroughJoinTest.java | 18 ++++------ .../PushDownMinMaxThroughJoinTest.java | 36 ++++--------------- .../rewrite/PushDownSumThroughJoinTest.java | 24 ++++--------- .../doris/utframe/TestWithFeService.java | 11 ++++++ 6 files changed, 45 insertions(+), 98 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 15e08d052e..183bfc3c91 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -91,9 +91,12 @@ import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderLimit; import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderTopN; import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoEsScan; import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoJdbcScan; +import org.apache.doris.nereids.rules.rewrite.PushDownCountThroughJoin; import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughProject; import org.apache.doris.nereids.rules.rewrite.PushDownLimit; import org.apache.doris.nereids.rules.rewrite.PushDownLimitDistinctThroughJoin; +import org.apache.doris.nereids.rules.rewrite.PushDownMinMaxThroughJoin; +import org.apache.doris.nereids.rules.rewrite.PushDownSumThroughJoin; import org.apache.doris.nereids.rules.rewrite.PushDownTopNThroughJoin; import org.apache.doris.nereids.rules.rewrite.PushDownTopNThroughWindow; import org.apache.doris.nereids.rules.rewrite.PushFilterInsideJoin; @@ -272,14 +275,13 @@ public class Rewriter extends AbstractBatchJobExecutor { topDown(new BuildAggForUnion()) ), - // topic("Eager aggregation", - // topDown( - // new PushDownSumThroughJoin(), - // new PushDownMinMaxThroughJoin(), - // new PushDownCountThroughJoin() - // ), - // custom(RuleType.PUSH_DOWN_DISTINCT_THROUGH_JOIN, PushDownDistinctThroughJoin::new) - // ), + topic("Eager aggregation", + topDown( + new PushDownSumThroughJoin(), + new PushDownMinMaxThroughJoin(), + new PushDownCountThroughJoin() + ) + ), topic("Limit optimization", // TODO: the logical plan should not contains any phase information, diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownCountThroughJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownCountThroughJoinTest.java index 21eebe1b37..34ccfe70f7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownCountThroughJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownCountThroughJoinTest.java @@ -42,15 +42,15 @@ import java.util.Set; class PushDownCountThroughJoinTest implements MemoPatternMatchSupported { private static final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private static final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); + private MockUp mockUp = new MockUp() { + @Mock + public Set getEnableNereidsRules() { + return ImmutableSet.of(RuleType.PUSH_DOWN_COUNT_THROUGH_JOIN.type()); + } + }; @Test void testSingleCount() { - new MockUp() { - @Mock - public Set getEnableNereidsRules() { - return ImmutableSet.of(RuleType.PUSH_DOWN_COUNT_THROUGH_JOIN.type()); - } - }; Alias count = new Count(scan1.getOutput().get(0)).alias("count"); LogicalPlan plan = new LogicalPlanBuilder(scan1) .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0)) @@ -71,12 +71,6 @@ class PushDownCountThroughJoinTest implements MemoPatternMatchSupported { @Test void testMultiCount() { - new MockUp() { - @Mock - public Set getEnableNereidsRules() { - return ImmutableSet.of(RuleType.PUSH_DOWN_COUNT_THROUGH_JOIN.type()); - } - }; Alias leftCnt1 = new Count(scan1.getOutput().get(0)).alias("leftCnt1"); Alias leftCnt2 = new Count(scan1.getOutput().get(1)).alias("leftCnt2"); Alias rightCnt1 = new Count(scan2.getOutput().get(1)).alias("rightCnt1"); @@ -100,12 +94,6 @@ class PushDownCountThroughJoinTest implements MemoPatternMatchSupported { @Test void testSingleCountStar() { - new MockUp() { - @Mock - public Set getEnableNereidsRules() { - return ImmutableSet.of(RuleType.PUSH_DOWN_COUNT_THROUGH_JOIN.type()); - } - }; Alias count = new Count().alias("countStar"); LogicalPlan plan = new LogicalPlanBuilder(scan1) .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0)) @@ -126,12 +114,6 @@ class PushDownCountThroughJoinTest implements MemoPatternMatchSupported { @Test void testSingleCountStarEmptyGroupBy() { - new MockUp() { - @Mock - public Set getEnableNereidsRules() { - return ImmutableSet.of(RuleType.PUSH_DOWN_COUNT_THROUGH_JOIN.type()); - } - }; Alias count = new Count().alias("countStar"); LogicalPlan plan = new LogicalPlanBuilder(scan1) .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0)) @@ -153,12 +135,6 @@ class PushDownCountThroughJoinTest implements MemoPatternMatchSupported { @Test void testBothSideCountAndCountStar() { - new MockUp() { - @Mock - public Set getEnableNereidsRules() { - return ImmutableSet.of(RuleType.PUSH_DOWN_COUNT_THROUGH_JOIN.type()); - } - }; Alias leftCnt = new Count(scan1.getOutput().get(0)).alias("leftCnt"); Alias rightCnt = new Count(scan2.getOutput().get(0)).alias("rightCnt"); Alias countStar = new Count().alias("countStar"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownDistinctThroughJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownDistinctThroughJoinTest.java index aac1bf2c7d..1ce91d1c7c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownDistinctThroughJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownDistinctThroughJoinTest.java @@ -42,15 +42,15 @@ class PushDownDistinctThroughJoinTest implements MemoPatternMatchSupported { private static final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); private static final LogicalOlapScan scan3 = PlanConstructor.newLogicalOlapScan(2, "t3", 0); private static final LogicalOlapScan scan4 = PlanConstructor.newLogicalOlapScan(3, "t4", 0); + private MockUp mockUp = new MockUp() { + @Mock + public Set getEnableNereidsRules() { + return ImmutableSet.of(RuleType.PUSH_DOWN_DISTINCT_THROUGH_JOIN.type()); + } + }; @Test void testPushdownJoin() { - new MockUp() { - @Mock - public Set getEnableNereidsRules() { - return ImmutableSet.of(RuleType.PUSH_DOWN_DISTINCT_THROUGH_JOIN.type()); - } - }; LogicalPlan plan = new LogicalPlanBuilder(scan1) .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0)) .join(scan3, JoinType.INNER_JOIN, Pair.of(0, 0)) @@ -73,12 +73,6 @@ class PushDownDistinctThroughJoinTest implements MemoPatternMatchSupported { @Test void testPushdownProjectJoin() { - new MockUp() { - @Mock - public Set getEnableNereidsRules() { - return ImmutableSet.of(RuleType.PUSH_DOWN_DISTINCT_THROUGH_JOIN.type()); - } - }; LogicalPlan plan = new LogicalPlanBuilder(scan1) .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0)) .project(ImmutableList.of(0, 2)) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownMinMaxThroughJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownMinMaxThroughJoinTest.java index dafef59100..cf28954a47 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownMinMaxThroughJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownMinMaxThroughJoinTest.java @@ -45,15 +45,15 @@ class PushDownMinMaxThroughJoinTest implements MemoPatternMatchSupported { private static final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); private static final LogicalOlapScan scan3 = PlanConstructor.newLogicalOlapScan(2, "t3", 0); private static final LogicalOlapScan scan4 = PlanConstructor.newLogicalOlapScan(3, "t4", 0); + private MockUp mockUp = new MockUp() { + @Mock + public Set getEnableNereidsRules() { + return ImmutableSet.of(RuleType.PUSH_DOWN_MIN_MAX_THROUGH_JOIN.type()); + } + }; @Test void testSingleJoin() { - new MockUp() { - @Mock - public Set getEnableNereidsRules() { - return ImmutableSet.of(RuleType.PUSH_DOWN_MIN_MAX_THROUGH_JOIN.type()); - } - }; Alias min = new Min(scan1.getOutput().get(0)).alias("min"); LogicalPlan plan = new LogicalPlanBuilder(scan1) .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0)) @@ -74,12 +74,6 @@ class PushDownMinMaxThroughJoinTest implements MemoPatternMatchSupported { @Test void testMultiJoin() { - new MockUp() { - @Mock - public Set getEnableNereidsRules() { - return ImmutableSet.of(RuleType.PUSH_DOWN_MIN_MAX_THROUGH_JOIN.type()); - } - }; Alias min = new Min(scan1.getOutput().get(0)).alias("min"); LogicalPlan plan = new LogicalPlanBuilder(scan1) .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0)) @@ -113,12 +107,6 @@ class PushDownMinMaxThroughJoinTest implements MemoPatternMatchSupported { @Test void testAggNotOutputGroupBy() { - new MockUp() { - @Mock - public Set getEnableNereidsRules() { - return ImmutableSet.of(RuleType.PUSH_DOWN_MIN_MAX_THROUGH_JOIN.type()); - } - }; // agg don't output group by Alias min = new Min(scan1.getOutput().get(0)).alias("min"); LogicalPlan plan = new LogicalPlanBuilder(scan1) @@ -146,12 +134,6 @@ class PushDownMinMaxThroughJoinTest implements MemoPatternMatchSupported { @Test void testBothSideSingleJoin() { - new MockUp() { - @Mock - public Set getEnableNereidsRules() { - return ImmutableSet.of(RuleType.PUSH_DOWN_MIN_MAX_THROUGH_JOIN.type()); - } - }; Alias min = new Min(scan1.getOutput().get(1)).alias("min"); Alias max = new Max(scan2.getOutput().get(1)).alias("max"); LogicalPlan plan = new LogicalPlanBuilder(scan1) @@ -174,12 +156,6 @@ class PushDownMinMaxThroughJoinTest implements MemoPatternMatchSupported { @Test void testBothSide() { - new MockUp() { - @Mock - public Set getEnableNereidsRules() { - return ImmutableSet.of(RuleType.PUSH_DOWN_MIN_MAX_THROUGH_JOIN.type()); - } - }; Alias min = new Min(scan1.getOutput().get(1)).alias("min"); Alias max = new Max(scan3.getOutput().get(1)).alias("max"); LogicalPlan plan = new LogicalPlanBuilder(scan1) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownSumThroughJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownSumThroughJoinTest.java index 5828d2319f..088372b0d7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownSumThroughJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownSumThroughJoinTest.java @@ -42,15 +42,15 @@ import java.util.Set; class PushDownSumThroughJoinTest implements MemoPatternMatchSupported { private static final LogicalOlapScan scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); private static final LogicalOlapScan scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); + private MockUp mockUp = new MockUp() { + @Mock + public Set getEnableNereidsRules() { + return ImmutableSet.of(RuleType.PUSH_DOWN_SUM_THROUGH_JOIN.type()); + } + }; @Test void testSingleJoinLeftSum() { - new MockUp() { - @Mock - public Set getEnableNereidsRules() { - return ImmutableSet.of(RuleType.PUSH_DOWN_SUM_THROUGH_JOIN.type()); - } - }; Alias sum = new Sum(scan1.getOutput().get(1)).alias("sum"); LogicalPlan plan = new LogicalPlanBuilder(scan1) .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0)) @@ -71,12 +71,6 @@ class PushDownSumThroughJoinTest implements MemoPatternMatchSupported { @Test void testSingleJoinRightSum() { - new MockUp() { - @Mock - public Set getEnableNereidsRules() { - return ImmutableSet.of(RuleType.PUSH_DOWN_SUM_THROUGH_JOIN.type()); - } - }; Alias sum = new Sum(scan2.getOutput().get(1)).alias("sum"); LogicalPlan plan = new LogicalPlanBuilder(scan1) .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0)) @@ -97,12 +91,6 @@ class PushDownSumThroughJoinTest implements MemoPatternMatchSupported { @Test void testAggNotOutputGroupBy() { - new MockUp() { - @Mock - public Set getEnableNereidsRules() { - return ImmutableSet.of(RuleType.PUSH_DOWN_SUM_THROUGH_JOIN.type()); - } - }; // agg don't output group by Alias sum = new Sum(scan1.getOutput().get(1)).alias("sum"); LogicalPlan plan = new LogicalPlanBuilder(scan1) diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java index 911155df7a..946e159529 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java @@ -69,6 +69,7 @@ import org.apache.doris.planner.Planner; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.OriginStatement; import org.apache.doris.qe.QueryState; +import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.ShowExecutor; import org.apache.doris.qe.ShowResultSet; import org.apache.doris.qe.StmtExecutor; @@ -84,9 +85,12 @@ import org.apache.doris.utframe.MockedFrontend.NotInitException; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import mockit.Mock; +import mockit.MockUp; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -129,6 +133,13 @@ public abstract class TestWithFeService { protected static final String DEFAULT_CLUSTER_PREFIX = "default_cluster:"; + private static final MockUp mockUp = new MockUp() { + @Mock + public Set getEnableNereidsRules() { + return ImmutableSet.of(); + } + }; + @BeforeAll public final void beforeAll() throws Exception { FeConstants.enableInternalSchemaDb = false;