From b45f501e51145de5c52d2ee0201175133f0317a6 Mon Sep 17 00:00:00 2001
From: JingDas <114388747+JingDas@users.noreply.github.com>
Date: Thu, 19 Oct 2023 12:07:37 +0800
Subject: [PATCH] [improvement](nereids) Support aggregate functions without
from clause (#25500)
Support aggregate functions in select without from clause, here are some examples as following:
SELECT 1,
'a',
COUNT(),
SUM(1) + 1,
AVG(2) / COUNT(),
MAX(3),
MIN(4),
RANK() OVER() AS w_rank,
DENSE_RANK() OVER() AS w_dense_rank,
ROW_NUMBER() OVER() AS w_row_number,
SUM(5) OVER() AS w_sum,
AVG(6) OVER() AS w_avg,
COUNT() OVER() AS w_count,
MAX(7) OVER() AS w_max,
MIN(8) OVER() AS w_min;
---
.../doris/nereids/jobs/executor/Analyzer.java | 4 +-
.../apache/doris/nereids/rules/RuleType.java | 1 +
.../nereids/rules/analysis/CheckAnalysis.java | 1 -
.../OneRowRelationExtractAggregate.java | 68 +++++++++++++++++++
.../analysis/ProjectToGlobalAggregate.java | 35 +---------
.../nereids/rules/rewrite/ColumnPruning.java | 3 +
.../visitor/ExpressionVisitors.java | 57 ++++++++++++++++
.../plans/logical/LogicalOneRowRelation.java | 4 --
.../select_no_from/sql/projectAggFuncs.out | 4 ++
.../select_no_from/sql/projectAggFuncs.out | 4 ++
.../select_no_from/sql/projectAggFuncs.sql | 19 +++++-
.../nereids_syntax_p0/one_row_relation.groovy | 7 --
.../select_no_from/sql/projectAggFuncs.sql | 18 ++++-
13 files changed, 173 insertions(+), 52 deletions(-)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/OneRowRelationExtractAggregate.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java
create mode 100644 regression-test/data/nereids_p0/select_no_from/sql/projectAggFuncs.out
create mode 100644 regression-test/data/query_p0/select_no_from/sql/projectAggFuncs.out
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java
index 333fc8a16d..e773ed41f6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java
@@ -34,6 +34,7 @@ import org.apache.doris.nereids.rules.analysis.EliminateGroupByConstant;
import org.apache.doris.nereids.rules.analysis.FillUpMissingSlots;
import org.apache.doris.nereids.rules.analysis.NormalizeAggregate;
import org.apache.doris.nereids.rules.analysis.NormalizeRepeat;
+import org.apache.doris.nereids.rules.analysis.OneRowRelationExtractAggregate;
import org.apache.doris.nereids.rules.analysis.ProjectToGlobalAggregate;
import org.apache.doris.nereids.rules.analysis.ProjectWithDistinctToAggregate;
import org.apache.doris.nereids.rules.analysis.ReplaceExpressionByChildOutput;
@@ -103,7 +104,8 @@ public class Analyzer extends AbstractBatchJobExecutor {
// please see rule BindSlotReference or BindFunction for example
new ProjectWithDistinctToAggregate(),
new ResolveOrdinalInOrderByAndGroupBy(),
- new ReplaceExpressionByChildOutput()
+ new ReplaceExpressionByChildOutput(),
+ new OneRowRelationExtractAggregate()
),
topDown(
new FillUpMissingSlots(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 617cb35290..f3b862f0c6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -71,6 +71,7 @@ public enum RuleType {
RESOLVE_PROJECT_ALIAS(RuleTypeClass.REWRITE),
RESOLVE_AGGREGATE_ALIAS(RuleTypeClass.REWRITE),
PROJECT_TO_GLOBAL_AGGREGATE(RuleTypeClass.REWRITE),
+ ONE_ROW_RELATION_EXTRACT_AGGREGATE(RuleTypeClass.REWRITE),
PROJECT_WITH_DISTINCT_TO_AGGREGATE(RuleTypeClass.REWRITE),
AVG_DISTINCT_TO_SUM_DIV_COUNT(RuleTypeClass.REWRITE),
ANALYZE_CTE(RuleTypeClass.REWRITE),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java
index d32823799e..fc6ddfcf79 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java
@@ -76,7 +76,6 @@ public class CheckAnalysis implements AnalysisRuleFactory {
TableGeneratingFunction.class,
WindowExpression.class))
.put(LogicalOneRowRelation.class, ImmutableSet.of(
- AggregateFunction.class,
GroupingScalarFunction.class,
TableGeneratingFunction.class,
WindowExpression.class))
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/OneRowRelationExtractAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/OneRowRelationExtractAggregate.java
new file mode 100644
index 0000000000..37aaf22ce7
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/OneRowRelationExtractAggregate.java
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.analysis;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitors;
+import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * OneRowRelationExtractAggregate.
+ *
+ * example sql:
+ *
+ * SELECT 1, 'a', COUNT();
+ *
+ *
+ * origin plan:
+ *
+ * LogicalOneRowRelation ( projects=[1 AS `1`#0, 'a' AS `'a'`#1, count(*) AS `count(*)`#2] )
+ * transformed plan:
+ *
+ * LogicalAggregate[23] ( groupByExpr=[], outputExpr=[1 AS `1`#0, 'a' AS `'a'`#1, count(*) AS `count(*)`#2],
+ * hasRepeat=false )
+ * LogicalOneRowRelation ( projects=[] )
+ */
+public class OneRowRelationExtractAggregate extends OneAnalysisRuleFactory {
+ @Override
+ public Rule build() {
+ return RuleType.ONE_ROW_RELATION_EXTRACT_AGGREGATE.build(
+ logicalOneRowRelation().then(relation -> {
+ List outputs = relation.getOutputs();
+ boolean needGlobalAggregate = outputs
+ .stream()
+ .anyMatch(p -> p.accept(ExpressionVisitors.CONTAINS_AGGREGATE_CHECKER, null));
+ if (needGlobalAggregate) {
+ LogicalRelation newRelation = new LogicalOneRowRelation(relation.getRelationId(),
+ ImmutableList.of());
+ return new LogicalAggregate<>(ImmutableList.of(), relation.getOutputs(), newRelation);
+ } else {
+ return relation;
+ }
+ })
+ );
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ProjectToGlobalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ProjectToGlobalAggregate.java
index 66371ae000..da642e7661 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ProjectToGlobalAggregate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ProjectToGlobalAggregate.java
@@ -19,10 +19,7 @@ package org.apache.doris.nereids.rules.analysis;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
-import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.WindowExpression;
-import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
-import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitors;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import com.google.common.collect.ImmutableList;
@@ -49,7 +46,7 @@ public class ProjectToGlobalAggregate extends OneAnalysisRuleFactory {
logicalProject().then(project -> {
boolean needGlobalAggregate = project.getProjects()
.stream()
- .anyMatch(p -> p.accept(ContainsAggregateChecker.INSTANCE, null));
+ .anyMatch(p -> p.accept(ExpressionVisitors.CONTAINS_AGGREGATE_CHECKER, null));
if (needGlobalAggregate) {
return new LogicalAggregate<>(ImmutableList.of(), project.getProjects(), project.child());
@@ -59,32 +56,4 @@ public class ProjectToGlobalAggregate extends OneAnalysisRuleFactory {
})
);
}
-
- private static class ContainsAggregateChecker extends DefaultExpressionVisitor {
-
- private static final ContainsAggregateChecker INSTANCE = new ContainsAggregateChecker();
-
- @Override
- public Boolean visit(Expression expr, Void context) {
- boolean needAggregate = false;
- for (Expression child : expr.children()) {
- needAggregate = needAggregate || child.accept(this, context);
- }
- return needAggregate;
- }
-
- @Override
- public Boolean visitWindow(WindowExpression windowExpression, Void context) {
- boolean needAggregate = false;
- for (Expression child : windowExpression.getExpressionsInWindowSpec()) {
- needAggregate = needAggregate || child.accept(this, context);
- }
- return needAggregate;
- }
-
- @Override
- public Boolean visitAggregateFunction(AggregateFunction aggregateFunction, Void context) {
- return true;
- }
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java
index 375ce34e6a..32e9cf64b6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ColumnPruning.java
@@ -215,6 +215,9 @@ public class ColumnPruning extends DefaultPlanRewriter implements
/** prune output */
public static P pruneOutput(P plan, List originOutput,
Function, P> withPrunedOutput, PruneContext context) {
+ if (originOutput.isEmpty()) {
+ return plan;
+ }
List prunedOutputs = originOutput.stream()
.filter(output -> context.requiredSlots.contains(output.toSlot()))
.collect(ImmutableList.toImmutableList());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java
new file mode 100644
index 0000000000..513da0e93d
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.visitor;
+
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.WindowExpression;
+import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
+
+/**
+ * This is the factory for all ExpressionVisitor instance.
+ * All children instance of DefaultExpressionVisitor or ExpressionVisitor for common usage
+ * should be here and expose self by class static final field.
+ */
+public class ExpressionVisitors {
+
+ public static final ContainsAggregateChecker CONTAINS_AGGREGATE_CHECKER = new ContainsAggregateChecker();
+
+ private static class ContainsAggregateChecker extends DefaultExpressionVisitor {
+ @Override
+ public Boolean visit(Expression expr, Void context) {
+ boolean needAggregate = false;
+ for (Expression child : expr.children()) {
+ needAggregate = needAggregate || child.accept(this, context);
+ }
+ return needAggregate;
+ }
+
+ @Override
+ public Boolean visitWindow(WindowExpression windowExpression, Void context) {
+ boolean needAggregate = false;
+ for (Expression child : windowExpression.getExpressionsInWindowSpec()) {
+ needAggregate = needAggregate || child.accept(this, context);
+ }
+ return needAggregate;
+ }
+
+ @Override
+ public Boolean visitAggregateFunction(AggregateFunction aggregateFunction, Void context) {
+ return true;
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java
index 68351c9a90..ffc7c0d625 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOneRowRelation.java
@@ -22,7 +22,6 @@ import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
-import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
@@ -30,7 +29,6 @@ import org.apache.doris.nereids.trees.plans.algebra.OneRowRelation;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
-import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import java.util.List;
@@ -52,8 +50,6 @@ public class LogicalOneRowRelation extends LogicalRelation implements OneRowRela
private LogicalOneRowRelation(RelationId relationId, List projects,
Optional groupExpression, Optional logicalProperties) {
super(relationId, PlanType.LOGICAL_ONE_ROW_RELATION, groupExpression, logicalProperties);
- Preconditions.checkArgument(projects.stream().noneMatch(p -> p.containsType(AggregateFunction.class)),
- "OneRowRelation can not contains any aggregate function");
this.projects = ImmutableList.copyOf(Objects.requireNonNull(projects, "projects can not be null"));
}
diff --git a/regression-test/data/nereids_p0/select_no_from/sql/projectAggFuncs.out b/regression-test/data/nereids_p0/select_no_from/sql/projectAggFuncs.out
new file mode 100644
index 0000000000..7ae8fd347e
--- /dev/null
+++ b/regression-test/data/nereids_p0/select_no_from/sql/projectAggFuncs.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !projectAggFuncs --
+1 a 1 2 2.0 3 4 1 1 1 5 6.0 1 7 8
+
diff --git a/regression-test/data/query_p0/select_no_from/sql/projectAggFuncs.out b/regression-test/data/query_p0/select_no_from/sql/projectAggFuncs.out
new file mode 100644
index 0000000000..7ae8fd347e
--- /dev/null
+++ b/regression-test/data/query_p0/select_no_from/sql/projectAggFuncs.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !projectAggFuncs --
+1 a 1 2 2.0 3 4 1 1 1 5 6.0 1 7 8
+
diff --git a/regression-test/suites/nereids_p0/select_no_from/sql/projectAggFuncs.sql b/regression-test/suites/nereids_p0/select_no_from/sql/projectAggFuncs.sql
index 50c3d56ed8..d2416eb991 100644
--- a/regression-test/suites/nereids_p0/select_no_from/sql/projectAggFuncs.sql
+++ b/regression-test/suites/nereids_p0/select_no_from/sql/projectAggFuncs.sql
@@ -1,4 +1,17 @@
-/*
-- database: presto; groups: no_from
-SELECT COUNT(10), MAX(50), MIN(90.0)
-*/
+SELECT 1,
+ 'a',
+ COUNT(),
+ SUM(1) + 1,
+ AVG(2) / COUNT(),
+ MAX(3),
+ MIN(4),
+ RANK() OVER() AS w_rank,
+ DENSE_RANK() OVER() AS w_dense_rank,
+ ROW_NUMBER() OVER() AS w_row_number,
+ SUM(5) OVER() AS w_sum,
+ AVG(6) OVER() AS w_avg,
+ COUNT() OVER() AS w_count,
+ MAX(7) OVER() AS w_max,
+ MIN(8) OVER() AS w_min;
+
diff --git a/regression-test/suites/nereids_syntax_p0/one_row_relation.groovy b/regression-test/suites/nereids_syntax_p0/one_row_relation.groovy
index 9b814cc4fe..e389960e79 100644
--- a/regression-test/suites/nereids_syntax_p0/one_row_relation.groovy
+++ b/regression-test/suites/nereids_syntax_p0/one_row_relation.groovy
@@ -31,11 +31,4 @@ suite("one_row_relation") {
)a"""
result([[100, "abc", "ab", "de", null]])
}
-
- test {
- sql """
- select sum(1);
- """
- exception "OneRowRelation can not contains any aggregate function"
- }
}
diff --git a/regression-test/suites/query_p0/select_no_from/sql/projectAggFuncs.sql b/regression-test/suites/query_p0/select_no_from/sql/projectAggFuncs.sql
index 50c3d56ed8..a0007ff2df 100644
--- a/regression-test/suites/query_p0/select_no_from/sql/projectAggFuncs.sql
+++ b/regression-test/suites/query_p0/select_no_from/sql/projectAggFuncs.sql
@@ -1,4 +1,16 @@
-/*
-- database: presto; groups: no_from
-SELECT COUNT(10), MAX(50), MIN(90.0)
-*/
+SELECT 1,
+ 'a',
+ COUNT(),
+ SUM(1) + 1,
+ AVG(2) / COUNT(),
+ MAX(3),
+ MIN(4),
+ RANK() OVER() AS w_rank,
+ DENSE_RANK() OVER() AS w_dense_rank,
+ ROW_NUMBER() OVER() AS w_row_number,
+ SUM(5) OVER() AS w_sum,
+ AVG(6) OVER() AS w_avg,
+ COUNT() OVER() AS w_count,
+ MAX(7) OVER() AS w_max,
+ MIN(8) OVER() AS w_min;